2024-12-09 19:03:13,263 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 19:03:13,277 main DEBUG Took 0.011952 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 19:03:13,278 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 19:03:13,278 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 19:03:13,279 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 19:03:13,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,289 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 19:03:13,301 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,303 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,303 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,304 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,305 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,306 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,306 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,307 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,308 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,308 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,309 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,310 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 19:03:13,310 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,310 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 19:03:13,312 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 19:03:13,313 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 19:03:13,314 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 19:03:13,315 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 19:03:13,316 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 19:03:13,316 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 19:03:13,323 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 19:03:13,326 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 19:03:13,327 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 19:03:13,328 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 19:03:13,328 main DEBUG createAppenders(={Console}) 2024-12-09 19:03:13,329 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-09 19:03:13,329 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 19:03:13,329 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-09 19:03:13,329 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 19:03:13,330 main DEBUG OutputStream closed 2024-12-09 19:03:13,330 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 19:03:13,330 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 19:03:13,330 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-09 19:03:13,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 19:03:13,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 19:03:13,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 19:03:13,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 19:03:13,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 19:03:13,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 19:03:13,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 19:03:13,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 19:03:13,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 19:03:13,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 19:03:13,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 19:03:13,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 19:03:13,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 19:03:13,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 19:03:13,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 19:03:13,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 19:03:13,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 19:03:13,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 19:03:13,411 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 19:03:13,412 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-09 19:03:13,412 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 19:03:13,413 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-09T19:03:13,603 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8 2024-12-09 19:03:13,606 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 19:03:13,606 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T19:03:13,615 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-09T19:03:13,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T19:03:13,646 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150, deleteOnExit=true 2024-12-09T19:03:13,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T19:03:13,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/test.cache.data in system properties and HBase conf 2024-12-09T19:03:13,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T19:03:13,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/hadoop.log.dir in system properties and HBase conf 2024-12-09T19:03:13,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T19:03:13,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T19:03:13,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T19:03:13,734 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T19:03:13,819 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T19:03:13,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T19:03:13,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T19:03:13,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T19:03:13,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T19:03:13,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T19:03:13,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T19:03:13,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T19:03:13,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T19:03:13,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T19:03:13,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/nfs.dump.dir in system properties and HBase conf 2024-12-09T19:03:13,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/java.io.tmpdir in system properties and HBase conf 2024-12-09T19:03:13,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T19:03:13,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T19:03:13,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T19:03:14,751 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T19:03:14,826 INFO [Time-limited test {}] log.Log(170): Logging initialized @2153ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T19:03:14,894 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T19:03:14,946 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T19:03:14,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T19:03:14,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T19:03:14,966 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T19:03:14,976 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T19:03:14,978 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/hadoop.log.dir/,AVAILABLE} 2024-12-09T19:03:14,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T19:03:15,140 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/java.io.tmpdir/jetty-localhost-43831-hadoop-hdfs-3_4_1-tests_jar-_-any-17835800870413648473/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T19:03:15,145 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43831} 2024-12-09T19:03:15,146 INFO [Time-limited test {}] server.Server(415): Started @2473ms 2024-12-09T19:03:15,591 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T19:03:15,597 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T19:03:15,598 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T19:03:15,598 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T19:03:15,598 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T19:03:15,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/hadoop.log.dir/,AVAILABLE} 2024-12-09T19:03:15,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T19:03:15,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/java.io.tmpdir/jetty-localhost-41659-hadoop-hdfs-3_4_1-tests_jar-_-any-8163823542688653105/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T19:03:15,696 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41659} 2024-12-09T19:03:15,697 INFO [Time-limited test {}] server.Server(415): Started @3024ms 2024-12-09T19:03:15,741 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T19:03:16,601 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/data/data1/current/BP-1730054910-172.17.0.2-1733770994308/current, will proceed with Du for space computation calculation, 2024-12-09T19:03:16,601 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/data/data2/current/BP-1730054910-172.17.0.2-1733770994308/current, will proceed with Du for space computation calculation, 2024-12-09T19:03:16,625 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T19:03:16,667 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x974a4f7a9034cd with lease ID 0xd339459c60d8fd3e: Processing first storage report for DS-4bfdddc1-3d95-4083-a285-0e7f73ce51af from datanode DatanodeRegistration(127.0.0.1:37109, datanodeUuid=ac1a6feb-a787-4ca5-acc6-45637ccb9f1c, infoPort=44275, infoSecurePort=0, ipcPort=38731, storageInfo=lv=-57;cid=testClusterID;nsid=523238835;c=1733770994308) 2024-12-09T19:03:16,668 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x974a4f7a9034cd with lease ID 0xd339459c60d8fd3e: from storage DS-4bfdddc1-3d95-4083-a285-0e7f73ce51af node DatanodeRegistration(127.0.0.1:37109, datanodeUuid=ac1a6feb-a787-4ca5-acc6-45637ccb9f1c, infoPort=44275, infoSecurePort=0, ipcPort=38731, storageInfo=lv=-57;cid=testClusterID;nsid=523238835;c=1733770994308), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T19:03:16,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x974a4f7a9034cd with lease ID 0xd339459c60d8fd3e: Processing first storage report for DS-68227d1e-ccad-4b6f-ac24-67a2c095da4e from datanode DatanodeRegistration(127.0.0.1:37109, datanodeUuid=ac1a6feb-a787-4ca5-acc6-45637ccb9f1c, infoPort=44275, infoSecurePort=0, ipcPort=38731, storageInfo=lv=-57;cid=testClusterID;nsid=523238835;c=1733770994308) 2024-12-09T19:03:16,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x974a4f7a9034cd with lease ID 0xd339459c60d8fd3e: from storage DS-68227d1e-ccad-4b6f-ac24-67a2c095da4e node DatanodeRegistration(127.0.0.1:37109, datanodeUuid=ac1a6feb-a787-4ca5-acc6-45637ccb9f1c, infoPort=44275, infoSecurePort=0, ipcPort=38731, storageInfo=lv=-57;cid=testClusterID;nsid=523238835;c=1733770994308), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T19:03:16,701 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8 2024-12-09T19:03:16,782 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/zookeeper_0, clientPort=60613, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T19:03:16,792 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60613 2024-12-09T19:03:16,800 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:16,802 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741825_1001 (size=7) 2024-12-09T19:03:17,407 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b with version=8 2024-12-09T19:03:17,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/hbase-staging 2024-12-09T19:03:17,486 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T19:03:17,702 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a437f8b9ba7d:0 server-side Connection retries=45 2024-12-09T19:03:17,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T19:03:17,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T19:03:17,714 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T19:03:17,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T19:03:17,714 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T19:03:17,830 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T19:03:17,881 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T19:03:17,889 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T19:03:17,893 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T19:03:17,913 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 102955 (auto-detected) 2024-12-09T19:03:17,913 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T19:03:17,929 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38053 2024-12-09T19:03:17,946 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38053 connecting to ZooKeeper ensemble=127.0.0.1:60613 2024-12-09T19:03:18,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380530x0, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T19:03:18,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38053-0x1000c2d089b0000 connected 2024-12-09T19:03:18,134 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:18,138 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:18,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T19:03:18,153 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b, hbase.cluster.distributed=false 2024-12-09T19:03:18,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T19:03:18,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38053 2024-12-09T19:03:18,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38053 2024-12-09T19:03:18,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38053 2024-12-09T19:03:18,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38053 2024-12-09T19:03:18,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38053 2024-12-09T19:03:18,267 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a437f8b9ba7d:0 server-side Connection retries=45 2024-12-09T19:03:18,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T19:03:18,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T19:03:18,269 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T19:03:18,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T19:03:18,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T19:03:18,271 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T19:03:18,273 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T19:03:18,274 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46189 2024-12-09T19:03:18,275 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46189 connecting to ZooKeeper ensemble=127.0.0.1:60613 2024-12-09T19:03:18,276 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:18,278 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:18,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461890x0, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T19:03:18,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461890x0, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T19:03:18,290 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46189-0x1000c2d089b0001 connected 2024-12-09T19:03:18,293 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T19:03:18,299 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T19:03:18,301 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T19:03:18,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T19:03:18,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46189 2024-12-09T19:03:18,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46189 2024-12-09T19:03:18,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46189 2024-12-09T19:03:18,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46189 2024-12-09T19:03:18,309 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46189 2024-12-09T19:03:18,321 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a437f8b9ba7d:38053 2024-12-09T19:03:18,322 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:18,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T19:03:18,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T19:03:18,332 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:18,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T19:03:18,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:18,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:18,356 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T19:03:18,357 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a437f8b9ba7d,38053,1733770997561 from backup master directory 2024-12-09T19:03:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T19:03:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T19:03:18,365 WARN [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T19:03:18,365 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:18,367 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T19:03:18,368 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T19:03:18,417 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/hbase.id] with ID: 8165842d-7141-4c13-bea5-3a7be743de1e 2024-12-09T19:03:18,418 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/.tmp/hbase.id 2024-12-09T19:03:18,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741826_1002 (size=42) 2024-12-09T19:03:18,833 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/.tmp/hbase.id]:[hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/hbase.id] 2024-12-09T19:03:18,880 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T19:03:18,886 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T19:03:18,903 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-09T19:03:18,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:18,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:18,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741827_1003 (size=196) 2024-12-09T19:03:19,393 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:03:19,395 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T19:03:19,409 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:19,413 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T19:03:19,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741828_1004 (size=1189) 2024-12-09T19:03:19,862 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store 2024-12-09T19:03:19,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741829_1005 (size=34) 2024-12-09T19:03:20,289 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T19:03:20,293 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:20,294 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T19:03:20,294 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:03:20,295 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:03:20,296 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T19:03:20,296 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:03:20,296 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:03:20,297 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733771000294Disabling compacts and flushes for region at 1733771000294Disabling writes for close at 1733771000296 (+2 ms)Writing region close event to WAL at 1733771000296Closed at 1733771000296 2024-12-09T19:03:20,299 WARN [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/.initializing 2024-12-09T19:03:20,299 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/WALs/a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:20,307 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T19:03:20,321 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C38053%2C1733770997561, suffix=, logDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/WALs/a437f8b9ba7d,38053,1733770997561, archiveDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/oldWALs, maxLogs=10 2024-12-09T19:03:20,341 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/WALs/a437f8b9ba7d,38053,1733770997561/a437f8b9ba7d%2C38053%2C1733770997561.1733771000325, exclude list is [], retry=0 2024-12-09T19:03:20,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37109,DS-4bfdddc1-3d95-4083-a285-0e7f73ce51af,DISK] 2024-12-09T19:03:20,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T19:03:20,387 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/WALs/a437f8b9ba7d,38053,1733770997561/a437f8b9ba7d%2C38053%2C1733770997561.1733771000325 2024-12-09T19:03:20,388 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275)] 2024-12-09T19:03:20,388 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:03:20,389 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:20,391 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,392 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T19:03:20,448 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:20,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:20,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T19:03:20,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:20,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:20,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T19:03:20,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:20,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:20,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T19:03:20,461 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:20,462 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:20,462 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,467 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,468 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,473 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,473 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,476 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T19:03:20,480 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T19:03:20,484 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:03:20,486 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71938423, jitterRate=0.07196603715419769}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T19:03:20,490 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733771000403Initializing all the Stores at 1733771000405 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771000406 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771000406Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771000407 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771000407Cleaning up temporary data from old regions at 1733771000473 (+66 ms)Region opened successfully at 1733771000490 (+17 ms) 2024-12-09T19:03:20,491 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T19:03:20,520 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d703cc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a437f8b9ba7d/172.17.0.2:0 2024-12-09T19:03:20,547 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T19:03:20,556 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T19:03:20,557 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T19:03:20,559 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T19:03:20,561 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T19:03:20,565 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T19:03:20,565 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T19:03:20,587 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T19:03:20,595 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T19:03:20,639 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T19:03:20,642 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T19:03:20,644 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T19:03:20,655 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T19:03:20,658 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T19:03:20,662 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T19:03:20,672 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T19:03:20,675 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T19:03:20,680 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T19:03:20,702 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T19:03:20,705 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T19:03:20,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T19:03:20,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T19:03:20,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:20,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:20,716 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a437f8b9ba7d,38053,1733770997561, sessionid=0x1000c2d089b0000, setting cluster-up flag (Was=false) 2024-12-09T19:03:20,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:20,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:20,764 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T19:03:20,769 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:20,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:20,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:20,813 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T19:03:20,815 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:20,822 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T19:03:20,884 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T19:03:20,894 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T19:03:20,900 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T19:03:20,905 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a437f8b9ba7d,38053,1733770997561 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T19:03:20,911 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T19:03:20,911 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T19:03:20,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T19:03:20,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T19:03:20,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a437f8b9ba7d:0, corePoolSize=10, maxPoolSize=10 2024-12-09T19:03:20,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:20,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=2, maxPoolSize=2 2024-12-09T19:03:20,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:20,915 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733771030915 2024-12-09T19:03:20,916 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(746): ClusterId : 8165842d-7141-4c13-bea5-3a7be743de1e 2024-12-09T19:03:20,917 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T19:03:20,917 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T19:03:20,917 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T19:03:20,918 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T19:03:20,919 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T19:03:20,922 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T19:03:20,922 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T19:03:20,922 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T19:03:20,922 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:20,922 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T19:03:20,923 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T19:03:20,923 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:20,926 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T19:03:20,928 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T19:03:20,928 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T19:03:20,930 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T19:03:20,930 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T19:03:20,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741831_1007 (size=1321) 2024-12-09T19:03:20,932 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.large.0-1733771000932,5,FailOnTimeoutGroup] 2024-12-09T19:03:20,932 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T19:03:20,932 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T19:03:20,933 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.small.0-1733771000932,5,FailOnTimeoutGroup] 2024-12-09T19:03:20,933 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:20,933 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T19:03:20,934 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:20,934 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:20,939 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T19:03:20,940 DEBUG [RS:0;a437f8b9ba7d:46189 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712ee775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a437f8b9ba7d/172.17.0.2:0 2024-12-09T19:03:20,952 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a437f8b9ba7d:46189 2024-12-09T19:03:20,954 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T19:03:20,955 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T19:03:20,955 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T19:03:20,957 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(2659): reportForDuty to master=a437f8b9ba7d,38053,1733770997561 with port=46189, startcode=1733770998237 2024-12-09T19:03:20,965 DEBUG [RS:0;a437f8b9ba7d:46189 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T19:03:20,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T19:03:20,998 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38053 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,000 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38053 {}] master.ServerManager(517): Registering regionserver=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,012 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:03:21,013 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34093 2024-12-09T19:03:21,013 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T19:03:21,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T19:03:21,023 DEBUG [RS:0;a437f8b9ba7d:46189 {}] zookeeper.ZKUtil(111): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,023 WARN [RS:0;a437f8b9ba7d:46189 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T19:03:21,023 INFO [RS:0;a437f8b9ba7d:46189 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T19:03:21,023 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,025 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a437f8b9ba7d,46189,1733770998237] 2024-12-09T19:03:21,049 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T19:03:21,059 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T19:03:21,064 INFO [RS:0;a437f8b9ba7d:46189 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T19:03:21,064 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,065 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T19:03:21,069 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T19:03:21,070 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,070 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=2, maxPoolSize=2 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,071 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,072 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,072 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,072 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,072 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T19:03:21,072 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T19:03:21,072 DEBUG [RS:0;a437f8b9ba7d:46189 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T19:03:21,073 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,073 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,073 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,074 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,074 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,074 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,46189,1733770998237-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T19:03:21,090 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T19:03:21,091 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,46189,1733770998237-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,092 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,092 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.Replication(171): a437f8b9ba7d,46189,1733770998237 started 2024-12-09T19:03:21,107 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:21,108 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1482): Serving as a437f8b9ba7d,46189,1733770998237, RpcServer on a437f8b9ba7d/172.17.0.2:46189, sessionid=0x1000c2d089b0001 2024-12-09T19:03:21,108 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T19:03:21,108 DEBUG [RS:0;a437f8b9ba7d:46189 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,109 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,46189,1733770998237' 2024-12-09T19:03:21,109 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T19:03:21,110 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T19:03:21,110 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T19:03:21,110 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T19:03:21,111 DEBUG [RS:0;a437f8b9ba7d:46189 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,111 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,46189,1733770998237' 2024-12-09T19:03:21,111 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T19:03:21,111 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T19:03:21,112 DEBUG [RS:0;a437f8b9ba7d:46189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T19:03:21,112 INFO [RS:0;a437f8b9ba7d:46189 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T19:03:21,112 INFO [RS:0;a437f8b9ba7d:46189 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T19:03:21,219 INFO [RS:0;a437f8b9ba7d:46189 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T19:03:21,223 INFO [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C46189%2C1733770998237, suffix=, logDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237, archiveDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/oldWALs, maxLogs=32 2024-12-09T19:03:21,237 DEBUG [RS:0;a437f8b9ba7d:46189 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237/a437f8b9ba7d%2C46189%2C1733770998237.1733771001224, exclude list is [], retry=0 2024-12-09T19:03:21,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37109,DS-4bfdddc1-3d95-4083-a285-0e7f73ce51af,DISK] 2024-12-09T19:03:21,244 INFO [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237/a437f8b9ba7d%2C46189%2C1733770998237.1733771001224 2024-12-09T19:03:21,245 DEBUG [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275)] 2024-12-09T19:03:21,334 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T19:03:21,335 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:03:21,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741833_1009 (size=32) 2024-12-09T19:03:21,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:21,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T19:03:21,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T19:03:21,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:21,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:21,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T19:03:21,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T19:03:21,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:21,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:21,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T19:03:21,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T19:03:21,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:21,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:21,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T19:03:21,777 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T19:03:21,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:21,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:21,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T19:03:21,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740 2024-12-09T19:03:21,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740 2024-12-09T19:03:21,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T19:03:21,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T19:03:21,784 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:03:21,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T19:03:21,791 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:03:21,792 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73281052, jitterRate=0.09197276830673218}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:03:21,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733771001756Initializing all the Stores at 1733771001761 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771001762 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771001762Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771001762Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771001762Cleaning up temporary data from old regions at 1733771001783 (+21 ms)Region opened successfully at 1733771001794 (+11 ms) 2024-12-09T19:03:21,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T19:03:21,794 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T19:03:21,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T19:03:21,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T19:03:21,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T19:03:21,795 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T19:03:21,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733771001794Disabling compacts and flushes for region at 1733771001794Disabling writes for close at 1733771001794Writing region close event to WAL at 1733771001795 (+1 ms)Closed at 1733771001795 2024-12-09T19:03:21,798 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T19:03:21,798 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T19:03:21,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T19:03:21,810 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T19:03:21,813 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T19:03:21,970 DEBUG [a437f8b9ba7d:38053 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T19:03:21,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:21,986 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a437f8b9ba7d,46189,1733770998237, state=OPENING 2024-12-09T19:03:21,997 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T19:03:22,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:22,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:03:22,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T19:03:22,007 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T19:03:22,009 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T19:03:22,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:03:22,196 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T19:03:22,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50727, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T19:03:22,211 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T19:03:22,211 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T19:03:22,212 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T19:03:22,215 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C46189%2C1733770998237.meta, suffix=.meta, logDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237, archiveDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/oldWALs, maxLogs=32 2024-12-09T19:03:22,228 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237/a437f8b9ba7d%2C46189%2C1733770998237.meta.1733771002216.meta, exclude list is [], retry=0 2024-12-09T19:03:22,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37109,DS-4bfdddc1-3d95-4083-a285-0e7f73ce51af,DISK] 2024-12-09T19:03:22,235 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/WALs/a437f8b9ba7d,46189,1733770998237/a437f8b9ba7d%2C46189%2C1733770998237.meta.1733771002216.meta 2024-12-09T19:03:22,235 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275)] 2024-12-09T19:03:22,235 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:03:22,237 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T19:03:22,239 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T19:03:22,243 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T19:03:22,247 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T19:03:22,248 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:22,248 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T19:03:22,248 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T19:03:22,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T19:03:22,252 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T19:03:22,253 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:22,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:22,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T19:03:22,255 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T19:03:22,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:22,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:22,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T19:03:22,257 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T19:03:22,257 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:22,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:22,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T19:03:22,260 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T19:03:22,260 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:22,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T19:03:22,261 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T19:03:22,262 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740 2024-12-09T19:03:22,265 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740 2024-12-09T19:03:22,267 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T19:03:22,267 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T19:03:22,268 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:03:22,271 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T19:03:22,273 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73999695, jitterRate=0.10268138349056244}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:03:22,273 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T19:03:22,274 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733771002248Writing region info on filesystem at 1733771002249 (+1 ms)Initializing all the Stores at 1733771002250 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771002250Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771002251 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771002251Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733771002251Cleaning up temporary data from old regions at 1733771002267 (+16 ms)Running coprocessor post-open hooks at 1733771002273 (+6 ms)Region opened successfully at 1733771002274 (+1 ms) 2024-12-09T19:03:22,280 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733771002188 2024-12-09T19:03:22,290 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T19:03:22,290 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T19:03:22,292 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:22,294 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a437f8b9ba7d,46189,1733770998237, state=OPEN 2024-12-09T19:03:22,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T19:03:22,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T19:03:22,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T19:03:22,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T19:03:22,324 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:22,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T19:03:22,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,46189,1733770998237 in 314 msec 2024-12-09T19:03:22,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T19:03:22,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 531 msec 2024-12-09T19:03:22,341 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T19:03:22,341 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T19:03:22,361 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:22,362 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:22,385 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:22,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48123, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:22,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5560 sec 2024-12-09T19:03:22,408 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733771002407, completionTime=-1 2024-12-09T19:03:22,410 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T19:03:22,410 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T19:03:22,434 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T19:03:22,434 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733771062434 2024-12-09T19:03:22,434 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733771122434 2024-12-09T19:03:22,434 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-09T19:03:22,436 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,38053,1733770997561-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,436 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,38053,1733770997561-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,436 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,38053,1733770997561-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,437 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a437f8b9ba7d:38053, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,438 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,438 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,443 DEBUG [master/a437f8b9ba7d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T19:03:22,462 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.097sec 2024-12-09T19:03:22,463 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T19:03:22,464 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T19:03:22,465 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T19:03:22,465 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T19:03:22,465 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T19:03:22,466 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,38053,1733770997561-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T19:03:22,466 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,38053,1733770997561-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T19:03:22,473 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T19:03:22,474 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T19:03:22,475 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,38053,1733770997561-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T19:03:22,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:22,533 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T19:03:22,533 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T19:03:22,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:22,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:22,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:22,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:22,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:22,561 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:22,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:22,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:22,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=2853 2024-12-09T19:03:22,606 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:03:22,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-12-09T19:03:22,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:03:22,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:03:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:03:22,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:03:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:22,655 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T19:03:22,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T19:03:22,658 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:22,661 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T19:03:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T19:03:22,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741835_1011 (size=959) 2024-12-09T19:03:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T19:03:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T19:03:23,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:03:23,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741836_1012 (size=53) 2024-12-09T19:03:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T19:03:23,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:23,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 07169db55e935a0918c76ee552d19727, disabling compactions & flushes 2024-12-09T19:03:23,489 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. after waiting 0 ms 2024-12-09T19:03:23,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,490 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07169db55e935a0918c76ee552d19727: Waiting for close lock at 1733771003489Disabling compacts and flushes for region at 1733771003489Disabling writes for close at 1733771003490 (+1 ms)Writing region close event to WAL at 1733771003490Closed at 1733771003490 2024-12-09T19:03:23,493 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T19:03:23,500 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733771003493"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733771003493"}]},"ts":"1733771003493"} 2024-12-09T19:03:23,505 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T19:03:23,507 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T19:03:23,509 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771003507"}]},"ts":"1733771003507"} 2024-12-09T19:03:23,513 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T19:03:23,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, ASSIGN}] 2024-12-09T19:03:23,517 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, ASSIGN 2024-12-09T19:03:23,519 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=false 2024-12-09T19:03:23,673 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=07169db55e935a0918c76ee552d19727, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:23,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, ASSIGN because future has completed 2024-12-09T19:03:23,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:03:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T19:03:23,849 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,850 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:03:23,850 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,850 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:23,851 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,851 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,854 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,856 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07169db55e935a0918c76ee552d19727 columnFamilyName A 2024-12-09T19:03:23,856 DEBUG [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:23,858 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] regionserver.HStore(327): Store=07169db55e935a0918c76ee552d19727/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:23,858 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,860 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07169db55e935a0918c76ee552d19727 columnFamilyName B 2024-12-09T19:03:23,860 DEBUG [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:23,861 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] regionserver.HStore(327): Store=07169db55e935a0918c76ee552d19727/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:23,861 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,864 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07169db55e935a0918c76ee552d19727 columnFamilyName C 2024-12-09T19:03:23,864 DEBUG [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:23,865 INFO [StoreOpener-07169db55e935a0918c76ee552d19727-1 {}] regionserver.HStore(327): Store=07169db55e935a0918c76ee552d19727/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:23,865 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,866 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,867 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,869 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,869 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,870 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:03:23,873 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,876 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:03:23,877 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 07169db55e935a0918c76ee552d19727; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59105976, jitterRate=-0.11925232410430908}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:03:23,877 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:23,879 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 07169db55e935a0918c76ee552d19727: Running coprocessor pre-open hook at 1733771003851Writing region info on filesystem at 1733771003851Initializing all the Stores at 1733771003853 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771003853Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771003853Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771003853Cleaning up temporary data from old regions at 1733771003869 (+16 ms)Running coprocessor post-open hooks at 1733771003877 (+8 ms)Region opened successfully at 1733771003879 (+2 ms) 2024-12-09T19:03:23,881 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., pid=6, masterSystemTime=1733771003840 2024-12-09T19:03:23,884 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,885 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:23,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=07169db55e935a0918c76ee552d19727, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:23,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:03:23,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T19:03:23,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 in 206 msec 2024-12-09T19:03:23,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T19:03:23,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, ASSIGN in 382 msec 2024-12-09T19:03:23,903 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T19:03:23,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771003903"}]},"ts":"1733771003903"} 2024-12-09T19:03:23,907 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T19:03:23,908 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T19:03:23,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2630 sec 2024-12-09T19:03:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T19:03:24,819 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:24,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,830 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77380ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a98157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,844 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:03:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53bd87e1 2024-12-09T19:03:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:03:24,849 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:03:24,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,870 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,890 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663b0bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643bd6db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,901 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee4cf7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,904 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b92dc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:24,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:24,908 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:24,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c944594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:24,916 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,916 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,916 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,916 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,917 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,917 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,917 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,918 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,918 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:24,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:24,928 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T19:03:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:03:24,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T19:03:24,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:24,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:25,014 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:25,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:03:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T19:03:25,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T19:03:25,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/0956ea5dd99c4f1a8334c352c5105ad0 is 50, key is test_row_0/A:col10/1733771004978/Put/seqid=0 2024-12-09T19:03:25,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741837_1013 (size=12001) 2024-12-09T19:03:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771065191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771065195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771065197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771065198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771065201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T19:03:25,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T19:03:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771065346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771065346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771065347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771065348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771065347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T19:03:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:25,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771065566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771065567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771065566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771065567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771065567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T19:03:25,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/0956ea5dd99c4f1a8334c352c5105ad0 2024-12-09T19:03:25,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T19:03:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/0f8b0063b1e14a2597cf4fe581e9e121 is 50, key is test_row_0/B:col10/1733771004978/Put/seqid=0 2024-12-09T19:03:25,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741838_1014 (size=12001) 2024-12-09T19:03:25,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/0f8b0063b1e14a2597cf4fe581e9e121 2024-12-09T19:03:25,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7 is 50, key is test_row_0/C:col10/1733771004978/Put/seqid=0 2024-12-09T19:03:25,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T19:03:25,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:25,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:25,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741839_1015 (size=12001) 2024-12-09T19:03:25,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7 2024-12-09T19:03:25,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/0956ea5dd99c4f1a8334c352c5105ad0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/0956ea5dd99c4f1a8334c352c5105ad0 2024-12-09T19:03:25,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/0956ea5dd99c4f1a8334c352c5105ad0, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T19:03:25,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/0f8b0063b1e14a2597cf4fe581e9e121 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/0f8b0063b1e14a2597cf4fe581e9e121 2024-12-09T19:03:25,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/0f8b0063b1e14a2597cf4fe581e9e121, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T19:03:25,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7 2024-12-09T19:03:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771065876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771065876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771065877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771065877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771065887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:25,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7, entries=150, sequenceid=14, filesize=11.7 K 2024-12-09T19:03:25,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 07169db55e935a0918c76ee552d19727 in 883ms, sequenceid=14, compaction requested=false 2024-12-09T19:03:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T19:03:25,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:25,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T19:03:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:25,943 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:03:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/012af5e7932a4cd49927a80a79f1b84a is 50, key is test_row_0/A:col10/1733771005188/Put/seqid=0 2024-12-09T19:03:25,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741840_1016 (size=12001) 2024-12-09T19:03:25,975 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/012af5e7932a4cd49927a80a79f1b84a 2024-12-09T19:03:26,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/a45c46e015e541248059de01ceffbc6e is 50, key is test_row_0/B:col10/1733771005188/Put/seqid=0 2024-12-09T19:03:26,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741841_1017 (size=12001) 2024-12-09T19:03:26,050 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/a45c46e015e541248059de01ceffbc6e 2024-12-09T19:03:26,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb is 50, key is test_row_0/C:col10/1733771005188/Put/seqid=0 2024-12-09T19:03:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T19:03:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741842_1018 (size=12001) 2024-12-09T19:03:26,104 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb 2024-12-09T19:03:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/012af5e7932a4cd49927a80a79f1b84a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/012af5e7932a4cd49927a80a79f1b84a 2024-12-09T19:03:26,138 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/012af5e7932a4cd49927a80a79f1b84a, entries=150, sequenceid=37, filesize=11.7 K 2024-12-09T19:03:26,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-12-09T19:03:26,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/a45c46e015e541248059de01ceffbc6e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a45c46e015e541248059de01ceffbc6e 2024-12-09T19:03:26,154 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a45c46e015e541248059de01ceffbc6e, entries=150, sequenceid=37, filesize=11.7 K 2024-12-09T19:03:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb 2024-12-09T19:03:26,176 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb, entries=150, sequenceid=37, filesize=11.7 K 2024-12-09T19:03:26,178 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 07169db55e935a0918c76ee552d19727 in 235ms, sequenceid=37, compaction requested=false 2024-12-09T19:03:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T19:03:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T19:03:26,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T19:03:26,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2320 sec 2024-12-09T19:03:26,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2610 sec 2024-12-09T19:03:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:26,421 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:26,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:03:26,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/7551ddc261c64959972e429e8180c223 is 50, key is test_row_0/A:col10/1733771006418/Put/seqid=0 2024-12-09T19:03:26,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741843_1019 (size=14341) 2024-12-09T19:03:26,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/7551ddc261c64959972e429e8180c223 2024-12-09T19:03:26,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/8c721fd7c63740979d5aabe3e4722318 is 50, key is test_row_0/B:col10/1733771006418/Put/seqid=0 2024-12-09T19:03:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771066477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771066481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771066481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771066485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771066489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741844_1020 (size=12001) 2024-12-09T19:03:26,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/8c721fd7c63740979d5aabe3e4722318 2024-12-09T19:03:26,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/66cb620be9b0468db60dc07a146cd803 is 50, key is test_row_0/C:col10/1733771006418/Put/seqid=0 2024-12-09T19:03:26,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741845_1021 (size=12001) 2024-12-09T19:03:26,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/66cb620be9b0468db60dc07a146cd803 2024-12-09T19:03:26,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/7551ddc261c64959972e429e8180c223 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/7551ddc261c64959972e429e8180c223 2024-12-09T19:03:26,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/7551ddc261c64959972e429e8180c223, entries=200, sequenceid=50, filesize=14.0 K 2024-12-09T19:03:26,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/8c721fd7c63740979d5aabe3e4722318 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8c721fd7c63740979d5aabe3e4722318 2024-12-09T19:03:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771066595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771066596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771066605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771066605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771066606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8c721fd7c63740979d5aabe3e4722318, entries=150, sequenceid=50, filesize=11.7 K 2024-12-09T19:03:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/66cb620be9b0468db60dc07a146cd803 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/66cb620be9b0468db60dc07a146cd803 2024-12-09T19:03:26,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/66cb620be9b0468db60dc07a146cd803, entries=150, sequenceid=50, filesize=11.7 K 2024-12-09T19:03:26,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 07169db55e935a0918c76ee552d19727 in 212ms, sequenceid=50, compaction requested=true 2024-12-09T19:03:26,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:26,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:26,641 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:26,641 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:26,646 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:26,646 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:26,648 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:26,648 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:26,649 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:26,650 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/0f8b0063b1e14a2597cf4fe581e9e121, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a45c46e015e541248059de01ceffbc6e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8c721fd7c63740979d5aabe3e4722318] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=35.2 K 2024-12-09T19:03:26,651 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:26,651 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/0956ea5dd99c4f1a8334c352c5105ad0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/012af5e7932a4cd49927a80a79f1b84a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/7551ddc261c64959972e429e8180c223] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=37.4 K 2024-12-09T19:03:26,652 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0956ea5dd99c4f1a8334c352c5105ad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733771004978 2024-12-09T19:03:26,652 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8b0063b1e14a2597cf4fe581e9e121, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733771004978 2024-12-09T19:03:26,653 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a45c46e015e541248059de01ceffbc6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733771005188 2024-12-09T19:03:26,653 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 012af5e7932a4cd49927a80a79f1b84a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733771005188 2024-12-09T19:03:26,654 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c721fd7c63740979d5aabe3e4722318, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733771006412 2024-12-09T19:03:26,654 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7551ddc261c64959972e429e8180c223, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733771006402 2024-12-09T19:03:26,705 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#9 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:26,706 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/74a665effd344a0997f903be8f54f3f9 is 50, key is test_row_0/B:col10/1733771006418/Put/seqid=0 2024-12-09T19:03:26,705 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#10 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:26,707 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3ef76fbacd6b484e898b034cef2e83e3 is 50, key is test_row_0/A:col10/1733771006418/Put/seqid=0 2024-12-09T19:03:26,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741846_1022 (size=12104) 2024-12-09T19:03:26,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741847_1023 (size=12104) 2024-12-09T19:03:26,745 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/74a665effd344a0997f903be8f54f3f9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/74a665effd344a0997f903be8f54f3f9 2024-12-09T19:03:26,755 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3ef76fbacd6b484e898b034cef2e83e3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ef76fbacd6b484e898b034cef2e83e3 2024-12-09T19:03:26,782 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 3ef76fbacd6b484e898b034cef2e83e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:26,782 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 74a665effd344a0997f903be8f54f3f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:26,782 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:26,782 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:26,784 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=13, startTime=1733771006638; duration=0sec 2024-12-09T19:03:26,784 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=13, startTime=1733771006636; duration=0sec 2024-12-09T19:03:26,785 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:26,785 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:26,785 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:26,785 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:26,786 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:26,789 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:26,789 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:26,789 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:26,789 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/66cb620be9b0468db60dc07a146cd803] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=35.2 K 2024-12-09T19:03:26,793 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ccf78e2cd9e48b193bfb34fc0e1b9a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733771004978 2024-12-09T19:03:26,794 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0af30f5a7844bd5b0a5cf4bc4a3c0eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733771005188 2024-12-09T19:03:26,795 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66cb620be9b0468db60dc07a146cd803, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733771006412 2024-12-09T19:03:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:26,808 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:26,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:03:26,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/b34f022e720d44618d30845e69cdbd60 is 50, key is test_row_0/A:col10/1733771006476/Put/seqid=0 2024-12-09T19:03:26,832 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#12 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:26,833 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/38d2739a147f4c8086abe300e2607144 is 50, key is test_row_0/C:col10/1733771006418/Put/seqid=0 2024-12-09T19:03:26,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771066833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771066834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771066837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771066841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771066842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741848_1024 (size=14341) 2024-12-09T19:03:26,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/b34f022e720d44618d30845e69cdbd60 2024-12-09T19:03:26,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741849_1025 (size=12104) 2024-12-09T19:03:26,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/13971cdad854490ea6b9b73d3563dfd6 is 50, key is test_row_0/B:col10/1733771006476/Put/seqid=0 2024-12-09T19:03:26,899 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/38d2739a147f4c8086abe300e2607144 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38d2739a147f4c8086abe300e2607144 2024-12-09T19:03:26,916 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 38d2739a147f4c8086abe300e2607144(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:26,916 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:26,916 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=13, startTime=1733771006638; duration=0sec 2024-12-09T19:03:26,916 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:26,916 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:26,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741850_1026 (size=12001) 2024-12-09T19:03:26,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/13971cdad854490ea6b9b73d3563dfd6 2024-12-09T19:03:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771066955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771066955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771066956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/919277c913e14084a64b0e07eae31bb2 is 50, key is test_row_0/C:col10/1733771006476/Put/seqid=0 2024-12-09T19:03:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771066956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771066956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741851_1027 (size=12001) 2024-12-09T19:03:26,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/919277c913e14084a64b0e07eae31bb2 2024-12-09T19:03:26,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/b34f022e720d44618d30845e69cdbd60 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b34f022e720d44618d30845e69cdbd60 2024-12-09T19:03:27,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b34f022e720d44618d30845e69cdbd60, entries=200, sequenceid=76, filesize=14.0 K 2024-12-09T19:03:27,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/13971cdad854490ea6b9b73d3563dfd6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/13971cdad854490ea6b9b73d3563dfd6 2024-12-09T19:03:27,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/13971cdad854490ea6b9b73d3563dfd6, entries=150, sequenceid=76, filesize=11.7 K 2024-12-09T19:03:27,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/919277c913e14084a64b0e07eae31bb2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/919277c913e14084a64b0e07eae31bb2 2024-12-09T19:03:27,044 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T19:03:27,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/919277c913e14084a64b0e07eae31bb2, entries=150, sequenceid=76, filesize=11.7 K 2024-12-09T19:03:27,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 07169db55e935a0918c76ee552d19727 in 255ms, sequenceid=76, compaction requested=false 2024-12-09T19:03:27,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T19:03:27,095 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:27,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T19:03:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T19:03:27,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:27,102 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:27,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:27,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:03:27,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/472f2f244e8a41b68f10320ac6bd1795 is 50, key is test_row_0/A:col10/1733771007167/Put/seqid=0 2024-12-09T19:03:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T19:03:27,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741852_1028 (size=16681) 2024-12-09T19:03:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771067219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771067220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771067223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771067227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771067227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771067334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771067335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771067334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771067346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771067345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T19:03:27,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:27,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:27,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771067545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771067545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771067545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771067555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771067555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:27,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/472f2f244e8a41b68f10320ac6bd1795 2024-12-09T19:03:27,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/743caed0b8c0484b99cd85cc0bd93f9b is 50, key is test_row_0/B:col10/1733771007167/Put/seqid=0 2024-12-09T19:03:27,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741853_1029 (size=12001) 2024-12-09T19:03:27,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/743caed0b8c0484b99cd85cc0bd93f9b 2024-12-09T19:03:27,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2addb27368014b8fbce37bcf4ba71e5b is 50, key is test_row_0/C:col10/1733771007167/Put/seqid=0 2024-12-09T19:03:27,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741854_1030 (size=12001) 2024-12-09T19:03:27,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2addb27368014b8fbce37bcf4ba71e5b 2024-12-09T19:03:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T19:03:27,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:27,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/472f2f244e8a41b68f10320ac6bd1795 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/472f2f244e8a41b68f10320ac6bd1795 2024-12-09T19:03:27,759 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:03:27,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/472f2f244e8a41b68f10320ac6bd1795, entries=250, sequenceid=92, filesize=16.3 K 2024-12-09T19:03:27,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/743caed0b8c0484b99cd85cc0bd93f9b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/743caed0b8c0484b99cd85cc0bd93f9b 2024-12-09T19:03:27,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/743caed0b8c0484b99cd85cc0bd93f9b, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:03:27,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2addb27368014b8fbce37bcf4ba71e5b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2addb27368014b8fbce37bcf4ba71e5b 2024-12-09T19:03:27,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2addb27368014b8fbce37bcf4ba71e5b, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:03:27,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 07169db55e935a0918c76ee552d19727 in 651ms, sequenceid=92, compaction requested=true 2024-12-09T19:03:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:27,823 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:27,823 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:27,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:27,825 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:27,825 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:27,825 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:27,825 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:27,825 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,825 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,825 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/74a665effd344a0997f903be8f54f3f9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/13971cdad854490ea6b9b73d3563dfd6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/743caed0b8c0484b99cd85cc0bd93f9b] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=35.3 K 2024-12-09T19:03:27,825 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ef76fbacd6b484e898b034cef2e83e3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b34f022e720d44618d30845e69cdbd60, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/472f2f244e8a41b68f10320ac6bd1795] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=42.1 K 2024-12-09T19:03:27,826 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef76fbacd6b484e898b034cef2e83e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733771006412 2024-12-09T19:03:27,826 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a665effd344a0997f903be8f54f3f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733771006412 2024-12-09T19:03:27,827 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b34f022e720d44618d30845e69cdbd60, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733771006476 2024-12-09T19:03:27,827 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13971cdad854490ea6b9b73d3563dfd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733771006476 2024-12-09T19:03:27,828 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 472f2f244e8a41b68f10320ac6bd1795, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771006833 2024-12-09T19:03:27,828 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743caed0b8c0484b99cd85cc0bd93f9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771007167 2024-12-09T19:03:27,852 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:27,853 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/28cce9a265b644d78fe8b94f6b5d2396 is 50, key is test_row_0/A:col10/1733771007167/Put/seqid=0 2024-12-09T19:03:27,855 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#19 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:27,856 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/778be0b8341b472d8dc192f0fb20aafa is 50, key is test_row_0/B:col10/1733771007167/Put/seqid=0 2024-12-09T19:03:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:27,862 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:27,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:03:27,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/b054e3bf524f485db4cfeeb671b319c8 is 50, key is test_row_0/A:col10/1733771007857/Put/seqid=0 2024-12-09T19:03:27,880 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T19:03:27,881 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T19:03:27,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741855_1031 (size=12207) 2024-12-09T19:03:27,883 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T19:03:27,883 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T19:03:27,883 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T19:03:27,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:27,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:27,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T19:03:27,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T19:03:27,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T19:03:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771067886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771067894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,905 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/28cce9a265b644d78fe8b94f6b5d2396 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/28cce9a265b644d78fe8b94f6b5d2396 2024-12-09T19:03:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771067895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771067898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771067901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:27,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741856_1032 (size=12207) 2024-12-09T19:03:27,922 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 28cce9a265b644d78fe8b94f6b5d2396(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:27,922 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:27,923 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=13, startTime=1733771007822; duration=0sec 2024-12-09T19:03:27,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:27,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:27,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741857_1033 (size=12001) 2024-12-09T19:03:27,935 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:27,935 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/778be0b8341b472d8dc192f0fb20aafa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/778be0b8341b472d8dc192f0fb20aafa 2024-12-09T19:03:27,935 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:27,935 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:27,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/b054e3bf524f485db4cfeeb671b319c8 2024-12-09T19:03:27,937 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38d2739a147f4c8086abe300e2607144, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/919277c913e14084a64b0e07eae31bb2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2addb27368014b8fbce37bcf4ba71e5b] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=35.3 K 2024-12-09T19:03:27,941 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d2739a147f4c8086abe300e2607144, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733771006412 2024-12-09T19:03:27,945 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 919277c913e14084a64b0e07eae31bb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733771006476 2024-12-09T19:03:27,948 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 778be0b8341b472d8dc192f0fb20aafa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:27,949 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:27,949 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=13, startTime=1733771007823; duration=0sec 2024-12-09T19:03:27,949 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:27,949 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:27,949 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2addb27368014b8fbce37bcf4ba71e5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771007167 2024-12-09T19:03:27,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/c322bb84102241e19e3713c073d10af6 is 50, key is test_row_0/B:col10/1733771007857/Put/seqid=0 2024-12-09T19:03:27,975 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:27,976 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2520baacc0ea44949be49cfd9c7dce25 is 50, key is test_row_0/C:col10/1733771007167/Put/seqid=0 2024-12-09T19:03:27,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741858_1034 (size=12001) 2024-12-09T19:03:27,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/c322bb84102241e19e3713c073d10af6 2024-12-09T19:03:28,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771068005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741859_1035 (size=12207) 2024-12-09T19:03:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771068016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771068016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771068016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771068016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/ce64e56f92d444c8ad1a2c75fc350386 is 50, key is test_row_0/C:col10/1733771007857/Put/seqid=0 2024-12-09T19:03:28,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:28,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:28,047 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2520baacc0ea44949be49cfd9c7dce25 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2520baacc0ea44949be49cfd9c7dce25 2024-12-09T19:03:28,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741860_1036 (size=12001) 2024-12-09T19:03:28,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/ce64e56f92d444c8ad1a2c75fc350386 2024-12-09T19:03:28,064 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 2520baacc0ea44949be49cfd9c7dce25(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:28,064 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:28,064 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=13, startTime=1733771007823; duration=0sec 2024-12-09T19:03:28,064 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:28,064 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/b054e3bf524f485db4cfeeb671b319c8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b054e3bf524f485db4cfeeb671b319c8 2024-12-09T19:03:28,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b054e3bf524f485db4cfeeb671b319c8, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:03:28,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/c322bb84102241e19e3713c073d10af6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/c322bb84102241e19e3713c073d10af6 2024-12-09T19:03:28,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/c322bb84102241e19e3713c073d10af6, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:03:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/ce64e56f92d444c8ad1a2c75fc350386 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/ce64e56f92d444c8ad1a2c75fc350386 2024-12-09T19:03:28,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/ce64e56f92d444c8ad1a2c75fc350386, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:03:28,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 07169db55e935a0918c76ee552d19727 in 270ms, sequenceid=115, compaction requested=false 2024-12-09T19:03:28,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:28,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T19:03:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:28,199 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:03:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4 is 50, key is test_row_0/A:col10/1733771007891/Put/seqid=0 2024-12-09T19:03:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:28,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741861_1037 (size=12001) 2024-12-09T19:03:28,233 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4 2024-12-09T19:03:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T19:03:28,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/21ef5d6acb1c4098800c2bbb0d157fc0 is 50, key is test_row_0/B:col10/1733771007891/Put/seqid=0 2024-12-09T19:03:28,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771068259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771068262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771068267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741862_1038 (size=12001) 2024-12-09T19:03:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771068270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771068270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771068375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771068375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771068384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771068384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771068385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771068584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771068584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771068595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771068595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771068595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,675 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/21ef5d6acb1c4098800c2bbb0d157fc0 2024-12-09T19:03:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/dc1dff43e16e4ac2b176bf5a0476314f is 50, key is test_row_0/C:col10/1733771007891/Put/seqid=0 2024-12-09T19:03:28,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741863_1039 (size=12001) 2024-12-09T19:03:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771068895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771068895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771068904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771068904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:28,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771068915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,102 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/dc1dff43e16e4ac2b176bf5a0476314f 2024-12-09T19:03:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4 2024-12-09T19:03:29,126 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4, entries=150, sequenceid=131, filesize=11.7 K 2024-12-09T19:03:29,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/21ef5d6acb1c4098800c2bbb0d157fc0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/21ef5d6acb1c4098800c2bbb0d157fc0 2024-12-09T19:03:29,147 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/21ef5d6acb1c4098800c2bbb0d157fc0, entries=150, sequenceid=131, filesize=11.7 K 2024-12-09T19:03:29,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/dc1dff43e16e4ac2b176bf5a0476314f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc1dff43e16e4ac2b176bf5a0476314f 2024-12-09T19:03:29,173 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc1dff43e16e4ac2b176bf5a0476314f, entries=150, sequenceid=131, filesize=11.7 K 2024-12-09T19:03:29,175 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 07169db55e935a0918c76ee552d19727 in 976ms, sequenceid=131, compaction requested=true 2024-12-09T19:03:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T19:03:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T19:03:29,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T19:03:29,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-12-09T19:03:29,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0890 sec 2024-12-09T19:03:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T19:03:29,245 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:29,245 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T19:03:29,250 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T19:03:29,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:29,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T19:03:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:29,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:29,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:03:29,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T19:03:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/048b7f4d523a45968e34edea5002ee34 is 50, key is test_row_0/A:col10/1733771009406/Put/seqid=0 2024-12-09T19:03:29,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771069424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771069425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771069426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771069428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771069430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741864_1040 (size=14541) 2024-12-09T19:03:29,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771069545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771069546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771069546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771069546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T19:03:29,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T19:03:29,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T19:03:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:29,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771069755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771069755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771069755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771069755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:29,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/048b7f4d523a45968e34edea5002ee34 2024-12-09T19:03:29,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/54cfc74fd33147d1afc787da15c07d14 is 50, key is test_row_0/B:col10/1733771009406/Put/seqid=0 2024-12-09T19:03:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T19:03:29,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T19:03:29,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:29,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:29,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741865_1041 (size=12151) 2024-12-09T19:03:29,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/54cfc74fd33147d1afc787da15c07d14 2024-12-09T19:03:29,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5c6760878402478aac33b2868fbc2810 is 50, key is test_row_0/C:col10/1733771009406/Put/seqid=0 2024-12-09T19:03:29,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741866_1042 (size=12151) 2024-12-09T19:03:29,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5c6760878402478aac33b2868fbc2810 2024-12-09T19:03:29,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/048b7f4d523a45968e34edea5002ee34 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/048b7f4d523a45968e34edea5002ee34 2024-12-09T19:03:29,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/048b7f4d523a45968e34edea5002ee34, entries=200, sequenceid=155, filesize=14.2 K 2024-12-09T19:03:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/54cfc74fd33147d1afc787da15c07d14 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54cfc74fd33147d1afc787da15c07d14 2024-12-09T19:03:29,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54cfc74fd33147d1afc787da15c07d14, entries=150, sequenceid=155, filesize=11.9 K 2024-12-09T19:03:29,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5c6760878402478aac33b2868fbc2810 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5c6760878402478aac33b2868fbc2810 2024-12-09T19:03:29,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5c6760878402478aac33b2868fbc2810, entries=150, sequenceid=155, filesize=11.9 K 2024-12-09T19:03:29,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 07169db55e935a0918c76ee552d19727 in 586ms, sequenceid=155, compaction requested=true 2024-12-09T19:03:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:29,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:29,994 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:03:29,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:29,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:29,994 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:03:29,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:29,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:29,996 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:03:29,996 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:29,996 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,996 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/778be0b8341b472d8dc192f0fb20aafa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/c322bb84102241e19e3713c073d10af6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/21ef5d6acb1c4098800c2bbb0d157fc0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54cfc74fd33147d1afc787da15c07d14] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=47.2 K 2024-12-09T19:03:29,997 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:03:29,997 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:29,997 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:29,998 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/28cce9a265b644d78fe8b94f6b5d2396, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b054e3bf524f485db4cfeeb671b319c8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/048b7f4d523a45968e34edea5002ee34] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=49.6 K 2024-12-09T19:03:29,998 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 778be0b8341b472d8dc192f0fb20aafa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771007167 2024-12-09T19:03:29,999 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28cce9a265b644d78fe8b94f6b5d2396, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771007167 2024-12-09T19:03:30,000 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting c322bb84102241e19e3713c073d10af6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771007219 2024-12-09T19:03:30,000 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b054e3bf524f485db4cfeeb671b319c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771007219 2024-12-09T19:03:30,002 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ef5d6acb1c4098800c2bbb0d157fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733771007878 2024-12-09T19:03:30,002 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd1e8304e5e43cd9f5ce4f261ff8eb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733771007878 2024-12-09T19:03:30,003 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 54cfc74fd33147d1afc787da15c07d14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733771008259 2024-12-09T19:03:30,003 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048b7f4d523a45968e34edea5002ee34, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733771008259 2024-12-09T19:03:30,027 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:30,028 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/44a336afdf5c44fe8907eca84e03a319 is 50, key is test_row_0/A:col10/1733771009406/Put/seqid=0 2024-12-09T19:03:30,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T19:03:30,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,034 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:03:30,044 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:30,045 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/8d0cc1a259674e86990405db3391beb4 is 50, key is test_row_0/B:col10/1733771009406/Put/seqid=0 2024-12-09T19:03:30,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741867_1043 (size=12493) 2024-12-09T19:03:30,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/9c703130c3f9434f951f50a244716d1d is 50, key is test_row_0/A:col10/1733771009425/Put/seqid=0 2024-12-09T19:03:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771070107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771070111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771070114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771070114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741869_1045 (size=12151) 2024-12-09T19:03:30,123 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/9c703130c3f9434f951f50a244716d1d 2024-12-09T19:03:30,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741868_1044 (size=12493) 2024-12-09T19:03:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/091755124fcb4716bb7e0a3ac7ff817c is 50, key is test_row_0/B:col10/1733771009425/Put/seqid=0 2024-12-09T19:03:30,140 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/8d0cc1a259674e86990405db3391beb4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8d0cc1a259674e86990405db3391beb4 2024-12-09T19:03:30,151 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 8d0cc1a259674e86990405db3391beb4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:30,151 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:30,151 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=12, startTime=1733771009994; duration=0sec 2024-12-09T19:03:30,152 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:30,152 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:30,152 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:03:30,155 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:03:30,155 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:30,155 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,155 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2520baacc0ea44949be49cfd9c7dce25, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/ce64e56f92d444c8ad1a2c75fc350386, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc1dff43e16e4ac2b176bf5a0476314f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5c6760878402478aac33b2868fbc2810] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=47.2 K 2024-12-09T19:03:30,156 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2520baacc0ea44949be49cfd9c7dce25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771007167 2024-12-09T19:03:30,157 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ce64e56f92d444c8ad1a2c75fc350386, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771007219 2024-12-09T19:03:30,158 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1dff43e16e4ac2b176bf5a0476314f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733771007878 2024-12-09T19:03:30,159 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c6760878402478aac33b2868fbc2810, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733771008259 2024-12-09T19:03:30,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741870_1046 (size=12151) 2024-12-09T19:03:30,186 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/091755124fcb4716bb7e0a3ac7ff817c 2024-12-09T19:03:30,197 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:30,198 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/4e00994d2de64ba8946162acde62e2f4 is 50, key is test_row_0/C:col10/1733771009406/Put/seqid=0 2024-12-09T19:03:30,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/dc546104223f40549a64f515fd27effa is 50, key is test_row_0/C:col10/1733771009425/Put/seqid=0 2024-12-09T19:03:30,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741871_1047 (size=12493) 2024-12-09T19:03:30,224 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/4e00994d2de64ba8946162acde62e2f4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/4e00994d2de64ba8946162acde62e2f4 2024-12-09T19:03:30,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771070225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771070225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771070225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771070226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741872_1048 (size=12151) 2024-12-09T19:03:30,233 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/dc546104223f40549a64f515fd27effa 2024-12-09T19:03:30,237 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 4e00994d2de64ba8946162acde62e2f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:30,238 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:30,238 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=12, startTime=1733771009994; duration=0sec 2024-12-09T19:03:30,238 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:30,238 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:30,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/9c703130c3f9434f951f50a244716d1d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/9c703130c3f9434f951f50a244716d1d 2024-12-09T19:03:30,254 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/9c703130c3f9434f951f50a244716d1d, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T19:03:30,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/091755124fcb4716bb7e0a3ac7ff817c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/091755124fcb4716bb7e0a3ac7ff817c 2024-12-09T19:03:30,268 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/091755124fcb4716bb7e0a3ac7ff817c, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T19:03:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/dc546104223f40549a64f515fd27effa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc546104223f40549a64f515fd27effa 2024-12-09T19:03:30,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc546104223f40549a64f515fd27effa, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T19:03:30,283 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 07169db55e935a0918c76ee552d19727 in 250ms, sequenceid=168, compaction requested=false 2024-12-09T19:03:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T19:03:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T19:03:30,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T19:03:30,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-12-09T19:03:30,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0470 sec 2024-12-09T19:03:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T19:03:30,387 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:30,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T19:03:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T19:03:30,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:30,393 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:30,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:30,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:30,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:03:30,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/6a807e84fb5146d9b814cdbbbe18cc71 is 50, key is test_row_0/A:col10/1733771010111/Put/seqid=0 2024-12-09T19:03:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771070448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771070449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771070449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771070452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771070452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741873_1049 (size=14541) 2024-12-09T19:03:30,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/6a807e84fb5146d9b814cdbbbe18cc71 2024-12-09T19:03:30,484 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/44a336afdf5c44fe8907eca84e03a319 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/44a336afdf5c44fe8907eca84e03a319 2024-12-09T19:03:30,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/df54bf1c7acb4604bec78a1c4417e764 is 50, key is test_row_0/B:col10/1733771010111/Put/seqid=0 2024-12-09T19:03:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T19:03:30,496 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 44a336afdf5c44fe8907eca84e03a319(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:30,496 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:30,496 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=12, startTime=1733771009993; duration=0sec 2024-12-09T19:03:30,497 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:30,497 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:30,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741874_1050 (size=12151) 2024-12-09T19:03:30,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771070565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771070565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771070566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771070566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771070566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T19:03:30,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771070775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771070775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771070775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771070775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771070775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:30,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:30,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:30,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:30,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/df54bf1c7acb4604bec78a1c4417e764 2024-12-09T19:03:30,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/38b09c3e04e3415ab09107485ea93a24 is 50, key is test_row_0/C:col10/1733771010111/Put/seqid=0 2024-12-09T19:03:30,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741875_1051 (size=12151) 2024-12-09T19:03:31,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T19:03:31,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771071084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771071084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771071084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771071085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771071085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:31,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/38b09c3e04e3415ab09107485ea93a24 2024-12-09T19:03:31,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/6a807e84fb5146d9b814cdbbbe18cc71 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/6a807e84fb5146d9b814cdbbbe18cc71 2024-12-09T19:03:31,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/6a807e84fb5146d9b814cdbbbe18cc71, entries=200, sequenceid=195, filesize=14.2 K 2024-12-09T19:03:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/df54bf1c7acb4604bec78a1c4417e764 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/df54bf1c7acb4604bec78a1c4417e764 2024-12-09T19:03:31,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/df54bf1c7acb4604bec78a1c4417e764, entries=150, sequenceid=195, filesize=11.9 K 2024-12-09T19:03:31,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/38b09c3e04e3415ab09107485ea93a24 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38b09c3e04e3415ab09107485ea93a24 2024-12-09T19:03:31,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38b09c3e04e3415ab09107485ea93a24, entries=150, sequenceid=195, filesize=11.9 K 2024-12-09T19:03:31,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 07169db55e935a0918c76ee552d19727 in 943ms, sequenceid=195, compaction requested=true 2024-12-09T19:03:31,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:31,382 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:31,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:31,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:31,382 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:31,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:31,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:31,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:31,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:31,383 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:31,383 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:31,383 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,384 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/44a336afdf5c44fe8907eca84e03a319, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/9c703130c3f9434f951f50a244716d1d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/6a807e84fb5146d9b814cdbbbe18cc71] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=38.3 K 2024-12-09T19:03:31,384 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a336afdf5c44fe8907eca84e03a319, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733771008259 2024-12-09T19:03:31,384 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:31,384 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:31,384 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,384 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c703130c3f9434f951f50a244716d1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733771009421 2024-12-09T19:03:31,384 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8d0cc1a259674e86990405db3391beb4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/091755124fcb4716bb7e0a3ac7ff817c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/df54bf1c7acb4604bec78a1c4417e764] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=35.9 K 2024-12-09T19:03:31,385 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a807e84fb5146d9b814cdbbbe18cc71, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771010095 2024-12-09T19:03:31,385 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d0cc1a259674e86990405db3391beb4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733771008259 2024-12-09T19:03:31,385 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 091755124fcb4716bb7e0a3ac7ff817c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733771009421 2024-12-09T19:03:31,385 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting df54bf1c7acb4604bec78a1c4417e764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771010111 2024-12-09T19:03:31,396 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:31,396 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:31,397 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/41561b2ff10c454cb0e73bac673a54cd is 50, key is test_row_0/B:col10/1733771010111/Put/seqid=0 2024-12-09T19:03:31,398 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3de66cfbc65440ffb5ccf9fee6a6650c is 50, key is test_row_0/A:col10/1733771010111/Put/seqid=0 2024-12-09T19:03:31,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741877_1053 (size=12595) 2024-12-09T19:03:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741876_1052 (size=12595) 2024-12-09T19:03:31,446 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3de66cfbc65440ffb5ccf9fee6a6650c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3de66cfbc65440ffb5ccf9fee6a6650c 2024-12-09T19:03:31,454 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 3de66cfbc65440ffb5ccf9fee6a6650c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:31,454 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:31,454 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=13, startTime=1733771011382; duration=0sec 2024-12-09T19:03:31,454 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:31,454 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:31,455 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:31,456 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:31,456 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:31,456 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,457 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/4e00994d2de64ba8946162acde62e2f4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc546104223f40549a64f515fd27effa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38b09c3e04e3415ab09107485ea93a24] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=35.9 K 2024-12-09T19:03:31,457 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e00994d2de64ba8946162acde62e2f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733771008259 2024-12-09T19:03:31,458 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc546104223f40549a64f515fd27effa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733771009421 2024-12-09T19:03:31,458 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b09c3e04e3415ab09107485ea93a24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771010111 2024-12-09T19:03:31,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T19:03:31,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,476 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:03:31,479 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:31,480 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5ce8cf1aa8ce46e39527f4e5e5987d77 is 50, key is test_row_0/C:col10/1733771010111/Put/seqid=0 2024-12-09T19:03:31,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/a3d3c5561cf64d3ea67b22f618e022cd is 50, key is test_row_0/A:col10/1733771010441/Put/seqid=0 2024-12-09T19:03:31,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741878_1054 (size=12595) 2024-12-09T19:03:31,514 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5ce8cf1aa8ce46e39527f4e5e5987d77 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5ce8cf1aa8ce46e39527f4e5e5987d77 2024-12-09T19:03:31,522 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 5ce8cf1aa8ce46e39527f4e5e5987d77(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:31,522 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:31,523 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=13, startTime=1733771011383; duration=0sec 2024-12-09T19:03:31,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:31,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T19:03:31,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741879_1055 (size=12151) 2024-12-09T19:03:31,529 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/a3d3c5561cf64d3ea67b22f618e022cd 2024-12-09T19:03:31,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/f31545e237634c378abaf8f60fea54b0 is 50, key is test_row_0/B:col10/1733771010441/Put/seqid=0 2024-12-09T19:03:31,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741880_1056 (size=12151) 2024-12-09T19:03:31,564 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/f31545e237634c378abaf8f60fea54b0 2024-12-09T19:03:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/821b845d906346e4b3c47f9851d47434 is 50, key is test_row_0/C:col10/1733771010441/Put/seqid=0 2024-12-09T19:03:31,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741881_1057 (size=12151) 2024-12-09T19:03:31,589 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/821b845d906346e4b3c47f9851d47434 2024-12-09T19:03:31,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/a3d3c5561cf64d3ea67b22f618e022cd as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a3d3c5561cf64d3ea67b22f618e022cd 2024-12-09T19:03:31,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a3d3c5561cf64d3ea67b22f618e022cd, entries=150, sequenceid=209, filesize=11.9 K 2024-12-09T19:03:31,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/f31545e237634c378abaf8f60fea54b0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/f31545e237634c378abaf8f60fea54b0 2024-12-09T19:03:31,635 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/f31545e237634c378abaf8f60fea54b0, entries=150, sequenceid=209, filesize=11.9 K 2024-12-09T19:03:31,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/821b845d906346e4b3c47f9851d47434 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/821b845d906346e4b3c47f9851d47434 2024-12-09T19:03:31,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771071625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,646 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/821b845d906346e4b3c47f9851d47434, entries=150, sequenceid=209, filesize=11.9 K 2024-12-09T19:03:31,648 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 07169db55e935a0918c76ee552d19727 in 172ms, sequenceid=209, compaction requested=false 2024-12-09T19:03:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T19:03:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:31,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:31,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:03:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T19:03:31,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T19:03:31,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2570 sec 2024-12-09T19:03:31,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2700 sec 2024-12-09T19:03:31,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/a423763158394855a895bd5274f4f559 is 50, key is test_row_0/A:col10/1733771011647/Put/seqid=0 2024-12-09T19:03:31,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771071664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771071664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771071662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771071672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741882_1058 (size=14541) 2024-12-09T19:03:31,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771071750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771071775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771071775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771071775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771071784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,836 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/41561b2ff10c454cb0e73bac673a54cd as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/41561b2ff10c454cb0e73bac673a54cd 2024-12-09T19:03:31,843 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 41561b2ff10c454cb0e73bac673a54cd(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:31,844 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:31,844 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=13, startTime=1733771011382; duration=0sec 2024-12-09T19:03:31,844 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:31,844 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:31,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771071955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771071984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771071984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771071984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771071995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/a423763158394855a895bd5274f4f559 2024-12-09T19:03:32,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/7626d34474f240f3addbcf8e8d417720 is 50, key is test_row_0/B:col10/1733771011647/Put/seqid=0 2024-12-09T19:03:32,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741883_1059 (size=12151) 2024-12-09T19:03:32,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/7626d34474f240f3addbcf8e8d417720 2024-12-09T19:03:32,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5b5686023284438cb15175f2fcf0b123 is 50, key is test_row_0/C:col10/1733771011647/Put/seqid=0 2024-12-09T19:03:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741884_1060 (size=12151) 2024-12-09T19:03:32,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771072264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771072294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771072295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771072295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771072304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5b5686023284438cb15175f2fcf0b123 2024-12-09T19:03:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T19:03:32,535 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:32,536 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T19:03:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T19:03:32,540 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:32,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/a423763158394855a895bd5274f4f559 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a423763158394855a895bd5274f4f559 2024-12-09T19:03:32,541 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:32,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:32,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a423763158394855a895bd5274f4f559, entries=200, sequenceid=235, filesize=14.2 K 2024-12-09T19:03:32,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/7626d34474f240f3addbcf8e8d417720 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7626d34474f240f3addbcf8e8d417720 2024-12-09T19:03:32,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7626d34474f240f3addbcf8e8d417720, entries=150, sequenceid=235, filesize=11.9 K 2024-12-09T19:03:32,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/5b5686023284438cb15175f2fcf0b123 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5b5686023284438cb15175f2fcf0b123 2024-12-09T19:03:32,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5b5686023284438cb15175f2fcf0b123, entries=150, sequenceid=235, filesize=11.9 K 2024-12-09T19:03:32,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 07169db55e935a0918c76ee552d19727 in 921ms, sequenceid=235, compaction requested=true 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:32,571 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:32,571 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:32,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:32,573 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:32,573 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:32,573 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:32,573 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:32,573 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:32,573 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:32,573 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3de66cfbc65440ffb5ccf9fee6a6650c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a3d3c5561cf64d3ea67b22f618e022cd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a423763158394855a895bd5274f4f559] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=38.4 K 2024-12-09T19:03:32,573 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/41561b2ff10c454cb0e73bac673a54cd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/f31545e237634c378abaf8f60fea54b0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7626d34474f240f3addbcf8e8d417720] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=36.0 K 2024-12-09T19:03:32,574 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 41561b2ff10c454cb0e73bac673a54cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771010111 2024-12-09T19:03:32,574 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de66cfbc65440ffb5ccf9fee6a6650c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771010111 2024-12-09T19:03:32,574 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f31545e237634c378abaf8f60fea54b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733771010441 2024-12-09T19:03:32,574 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d3c5561cf64d3ea67b22f618e022cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733771010441 2024-12-09T19:03:32,575 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting a423763158394855a895bd5274f4f559, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733771011622 2024-12-09T19:03:32,575 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 7626d34474f240f3addbcf8e8d417720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733771011622 2024-12-09T19:03:32,589 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:32,590 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/56f82e1e5d6241adac0371caf8181bd5 is 50, key is test_row_0/B:col10/1733771011647/Put/seqid=0 2024-12-09T19:03:32,597 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:32,598 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/49a8c8bab9144a0faafcfa17b36cf04f is 50, key is test_row_0/A:col10/1733771011647/Put/seqid=0 2024-12-09T19:03:32,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741886_1062 (size=12697) 2024-12-09T19:03:32,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741885_1061 (size=12697) 2024-12-09T19:03:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T19:03:32,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T19:03:32,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:32,694 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:03:32,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/1b7d226672ae4c00bfcdc4629dc9fa07 is 50, key is test_row_0/A:col10/1733771011669/Put/seqid=0 2024-12-09T19:03:32,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741887_1063 (size=12151) 2024-12-09T19:03:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:32,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771072822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771072824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771072825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771072830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771072831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T19:03:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771072934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771072935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771072935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771072944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771072944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,034 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/49a8c8bab9144a0faafcfa17b36cf04f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/49a8c8bab9144a0faafcfa17b36cf04f 2024-12-09T19:03:33,036 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/56f82e1e5d6241adac0371caf8181bd5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/56f82e1e5d6241adac0371caf8181bd5 2024-12-09T19:03:33,043 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 49a8c8bab9144a0faafcfa17b36cf04f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:33,044 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:33,044 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=13, startTime=1733771012571; duration=0sec 2024-12-09T19:03:33,044 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:33,044 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:33,044 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:33,045 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:33,045 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:33,046 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:33,046 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5ce8cf1aa8ce46e39527f4e5e5987d77, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/821b845d906346e4b3c47f9851d47434, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5b5686023284438cb15175f2fcf0b123] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=36.0 K 2024-12-09T19:03:33,046 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce8cf1aa8ce46e39527f4e5e5987d77, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771010111 2024-12-09T19:03:33,047 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 56f82e1e5d6241adac0371caf8181bd5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:33,047 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 821b845d906346e4b3c47f9851d47434, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733771010441 2024-12-09T19:03:33,047 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:33,047 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=13, startTime=1733771012571; duration=0sec 2024-12-09T19:03:33,047 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b5686023284438cb15175f2fcf0b123, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733771011622 2024-12-09T19:03:33,048 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:33,048 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:33,056 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:33,057 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/52079ba639f84ff7b5bd9a75f883fa93 is 50, key is test_row_0/C:col10/1733771011647/Put/seqid=0 2024-12-09T19:03:33,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741888_1064 (size=12697) 2024-12-09T19:03:33,090 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/52079ba639f84ff7b5bd9a75f883fa93 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/52079ba639f84ff7b5bd9a75f883fa93 2024-12-09T19:03:33,098 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 52079ba639f84ff7b5bd9a75f883fa93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:33,098 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:33,098 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=13, startTime=1733771012571; duration=0sec 2024-12-09T19:03:33,099 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:33,099 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:33,108 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/1b7d226672ae4c00bfcdc4629dc9fa07 2024-12-09T19:03:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/270d4dc8c97447dc9f3e6ae0a72c6ee8 is 50, key is test_row_0/B:col10/1733771011669/Put/seqid=0 2024-12-09T19:03:33,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741889_1065 (size=12151) 2024-12-09T19:03:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771073144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771073144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771073144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771073155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771073155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T19:03:33,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771073461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771073461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771073461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771073465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771073465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,525 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/270d4dc8c97447dc9f3e6ae0a72c6ee8 2024-12-09T19:03:33,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2bf31a9ae5d347e1955fcff26c9c5114 is 50, key is test_row_0/C:col10/1733771011669/Put/seqid=0 2024-12-09T19:03:33,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741890_1066 (size=12151) 2024-12-09T19:03:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T19:03:33,948 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2bf31a9ae5d347e1955fcff26c9c5114 2024-12-09T19:03:33,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/1b7d226672ae4c00bfcdc4629dc9fa07 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/1b7d226672ae4c00bfcdc4629dc9fa07 2024-12-09T19:03:33,962 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/1b7d226672ae4c00bfcdc4629dc9fa07, entries=150, sequenceid=247, filesize=11.9 K 2024-12-09T19:03:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/270d4dc8c97447dc9f3e6ae0a72c6ee8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/270d4dc8c97447dc9f3e6ae0a72c6ee8 2024-12-09T19:03:33,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771073964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,974 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/270d4dc8c97447dc9f3e6ae0a72c6ee8, entries=150, sequenceid=247, filesize=11.9 K 2024-12-09T19:03:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2bf31a9ae5d347e1955fcff26c9c5114 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2bf31a9ae5d347e1955fcff26c9c5114 2024-12-09T19:03:33,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771073975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771073975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771073975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771073975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:33,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2bf31a9ae5d347e1955fcff26c9c5114, entries=150, sequenceid=247, filesize=11.9 K 2024-12-09T19:03:33,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 07169db55e935a0918c76ee552d19727 in 1291ms, sequenceid=247, compaction requested=false 2024-12-09T19:03:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T19:03:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T19:03:33,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T19:03:33,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-12-09T19:03:33,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4570 sec 2024-12-09T19:03:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T19:03:34,685 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:34,685 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T19:03:34,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T19:03:34,688 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:34,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T19:03:34,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T19:03:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:34,842 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-09T19:03:34,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3122bce69c8f48659f6696d147b3b843 is 50, key is test_row_0/A:col10/1733771012829/Put/seqid=0 2024-12-09T19:03:34,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741891_1067 (size=12301) 2024-12-09T19:03:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:34,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771074991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771074992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:34,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771074994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:34,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771074994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:34,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771074995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:34,999 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:34,999 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:34,999 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T19:03:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771075105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771075105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,255 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3122bce69c8f48659f6696d147b3b843 2024-12-09T19:03:35,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/fc4fad72368d47b7be6f0fa752cb972f is 50, key is test_row_0/B:col10/1733771012829/Put/seqid=0 2024-12-09T19:03:35,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741892_1068 (size=12301) 2024-12-09T19:03:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T19:03:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771075314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771075314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771075625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771075625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:35,672 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/fc4fad72368d47b7be6f0fa752cb972f 2024-12-09T19:03:35,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/a2e8db48820c4f5892dde9ad791294f2 is 50, key is test_row_0/C:col10/1733771012829/Put/seqid=0 2024-12-09T19:03:35,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741893_1069 (size=12301) 2024-12-09T19:03:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T19:03:36,091 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/a2e8db48820c4f5892dde9ad791294f2 2024-12-09T19:03:36,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3122bce69c8f48659f6696d147b3b843 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3122bce69c8f48659f6696d147b3b843 2024-12-09T19:03:36,103 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3122bce69c8f48659f6696d147b3b843, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T19:03:36,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/fc4fad72368d47b7be6f0fa752cb972f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/fc4fad72368d47b7be6f0fa752cb972f 2024-12-09T19:03:36,111 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/fc4fad72368d47b7be6f0fa752cb972f, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T19:03:36,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/a2e8db48820c4f5892dde9ad791294f2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/a2e8db48820c4f5892dde9ad791294f2 2024-12-09T19:03:36,118 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/a2e8db48820c4f5892dde9ad791294f2, entries=150, sequenceid=276, filesize=12.0 K 2024-12-09T19:03:36,120 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 07169db55e935a0918c76ee552d19727 in 1277ms, sequenceid=276, compaction requested=true 2024-12-09T19:03:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T19:03:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T19:03:36,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T19:03:36,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4330 sec 2024-12-09T19:03:36,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4390 sec 2024-12-09T19:03:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:36,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:36,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:03:36,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/98ac836212744f98a52a83f487db221c is 50, key is test_row_0/A:col10/1733771016135/Put/seqid=0 2024-12-09T19:03:36,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741894_1070 (size=12301) 2024-12-09T19:03:36,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771076185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771076190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771076295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771076295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771076504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771076504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/98ac836212744f98a52a83f487db221c 2024-12-09T19:03:36,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/54b8b4a2cdf14904a58d2d9a1dbda20f is 50, key is test_row_0/B:col10/1733771016135/Put/seqid=0 2024-12-09T19:03:36,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741895_1071 (size=12301) 2024-12-09T19:03:36,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771076815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771076815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T19:03:36,845 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:36,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T19:03:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T19:03:36,848 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:36,848 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:36,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T19:03:36,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/54b8b4a2cdf14904a58d2d9a1dbda20f 2024-12-09T19:03:36,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f564afdaafca42abacde3f244b10e3fe is 50, key is test_row_0/C:col10/1733771016135/Put/seqid=0 2024-12-09T19:03:36,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741896_1072 (size=12301) 2024-12-09T19:03:36,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f564afdaafca42abacde3f244b10e3fe 2024-12-09T19:03:37,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/98ac836212744f98a52a83f487db221c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/98ac836212744f98a52a83f487db221c 2024-12-09T19:03:37,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/98ac836212744f98a52a83f487db221c, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T19:03:37,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/54b8b4a2cdf14904a58d2d9a1dbda20f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54b8b4a2cdf14904a58d2d9a1dbda20f 2024-12-09T19:03:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771077014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,018 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54b8b4a2cdf14904a58d2d9a1dbda20f, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T19:03:37,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f564afdaafca42abacde3f244b10e3fe as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f564afdaafca42abacde3f244b10e3fe 2024-12-09T19:03:37,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f564afdaafca42abacde3f244b10e3fe, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T19:03:37,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 07169db55e935a0918c76ee552d19727 in 894ms, sequenceid=288, compaction requested=true 2024-12-09T19:03:37,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:37,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:37,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:37,030 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:03:37,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:37,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:37,031 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:03:37,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:37,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:37,033 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:37,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-09T19:03:37,035 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:03:37,035 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:03:37,035 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:37,035 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:37,036 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,036 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,036 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/56f82e1e5d6241adac0371caf8181bd5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/270d4dc8c97447dc9f3e6ae0a72c6ee8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/fc4fad72368d47b7be6f0fa752cb972f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54b8b4a2cdf14904a58d2d9a1dbda20f] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=48.3 K 2024-12-09T19:03:37,036 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/49a8c8bab9144a0faafcfa17b36cf04f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/1b7d226672ae4c00bfcdc4629dc9fa07, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3122bce69c8f48659f6696d147b3b843, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/98ac836212744f98a52a83f487db221c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=48.3 K 2024-12-09T19:03:37,036 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f82e1e5d6241adac0371caf8181bd5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733771011622 2024-12-09T19:03:37,037 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a8c8bab9144a0faafcfa17b36cf04f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733771011622 2024-12-09T19:03:37,039 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 270d4dc8c97447dc9f3e6ae0a72c6ee8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733771011659 2024-12-09T19:03:37,040 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b7d226672ae4c00bfcdc4629dc9fa07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733771011659 2024-12-09T19:03:37,040 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4fad72368d47b7be6f0fa752cb972f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733771012815 2024-12-09T19:03:37,040 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3122bce69c8f48659f6696d147b3b843, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733771012815 2024-12-09T19:03:37,041 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b8b4a2cdf14904a58d2d9a1dbda20f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733771014984 2024-12-09T19:03:37,042 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ac836212744f98a52a83f487db221c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733771014984 2024-12-09T19:03:37,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3ed4491b801b42d496ae76021a46b4b4 is 50, key is test_row_0/A:col10/1733771016183/Put/seqid=0 2024-12-09T19:03:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771077058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741897_1073 (size=12301) 2024-12-09T19:03:37,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771077060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3ed4491b801b42d496ae76021a46b4b4 2024-12-09T19:03:37,075 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:37,076 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/9ac9a4baa73641b48feda8cd7c8458dc is 50, key is test_row_0/B:col10/1733771016135/Put/seqid=0 2024-12-09T19:03:37,081 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:37,082 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/838fcb1293fc4065bb77c9167895a81c is 50, key is test_row_0/A:col10/1733771016135/Put/seqid=0 2024-12-09T19:03:37,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/7f6f13a589c14e5da182e8e8cc13b311 is 50, key is test_row_0/B:col10/1733771016183/Put/seqid=0 2024-12-09T19:03:37,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741898_1074 (size=12983) 2024-12-09T19:03:37,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741899_1075 (size=12983) 2024-12-09T19:03:37,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741900_1076 (size=12301) 2024-12-09T19:03:37,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T19:03:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771077165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771077175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771077324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771077334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771077375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771077384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T19:03:37,507 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/9ac9a4baa73641b48feda8cd7c8458dc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/9ac9a4baa73641b48feda8cd7c8458dc 2024-12-09T19:03:37,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/7f6f13a589c14e5da182e8e8cc13b311 2024-12-09T19:03:37,514 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/838fcb1293fc4065bb77c9167895a81c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/838fcb1293fc4065bb77c9167895a81c 2024-12-09T19:03:37,517 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 9ac9a4baa73641b48feda8cd7c8458dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:37,518 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:37,518 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=12, startTime=1733771017030; duration=0sec 2024-12-09T19:03:37,518 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:37,518 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:37,518 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:03:37,521 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:03:37,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/e450870015784a67b277e4cce289de9e is 50, key is test_row_0/C:col10/1733771016183/Put/seqid=0 2024-12-09T19:03:37,521 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:37,521 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,521 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/52079ba639f84ff7b5bd9a75f883fa93, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2bf31a9ae5d347e1955fcff26c9c5114, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/a2e8db48820c4f5892dde9ad791294f2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f564afdaafca42abacde3f244b10e3fe] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=48.3 K 2024-12-09T19:03:37,521 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 838fcb1293fc4065bb77c9167895a81c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:37,522 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:37,522 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=12, startTime=1733771017030; duration=0sec 2024-12-09T19:03:37,522 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:37,522 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:37,522 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 52079ba639f84ff7b5bd9a75f883fa93, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733771011622 2024-12-09T19:03:37,523 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf31a9ae5d347e1955fcff26c9c5114, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733771011659 2024-12-09T19:03:37,523 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e8db48820c4f5892dde9ad791294f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733771012815 2024-12-09T19:03:37,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f564afdaafca42abacde3f244b10e3fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733771014984 2024-12-09T19:03:37,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741901_1077 (size=12301) 2024-12-09T19:03:37,533 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:37,534 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/8b87dadf97c64becaead04c561f070c5 is 50, key is test_row_0/C:col10/1733771016135/Put/seqid=0 2024-12-09T19:03:37,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741902_1078 (size=12983) 2024-12-09T19:03:37,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771077684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771077694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:37,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/e450870015784a67b277e4cce289de9e 2024-12-09T19:03:37,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/3ed4491b801b42d496ae76021a46b4b4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ed4491b801b42d496ae76021a46b4b4 2024-12-09T19:03:37,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:37,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:37,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:37,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:37,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ed4491b801b42d496ae76021a46b4b4, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T19:03:37,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/7f6f13a589c14e5da182e8e8cc13b311 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7f6f13a589c14e5da182e8e8cc13b311 2024-12-09T19:03:37,947 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/8b87dadf97c64becaead04c561f070c5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/8b87dadf97c64becaead04c561f070c5 2024-12-09T19:03:37,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7f6f13a589c14e5da182e8e8cc13b311, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T19:03:37,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/e450870015784a67b277e4cce289de9e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e450870015784a67b277e4cce289de9e 2024-12-09T19:03:37,955 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 8b87dadf97c64becaead04c561f070c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:37,955 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:37,955 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=12, startTime=1733771017031; duration=0sec 2024-12-09T19:03:37,955 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:37,955 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:37,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e450870015784a67b277e4cce289de9e, entries=150, sequenceid=314, filesize=12.0 K 2024-12-09T19:03:37,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 07169db55e935a0918c76ee552d19727 in 926ms, sequenceid=314, compaction requested=false 2024-12-09T19:03:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T19:03:38,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T19:03:38,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:38,090 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-09T19:03:38,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/bdf1966cbcc24a27bce100e8aad5e9c0 is 50, key is test_row_0/A:col10/1733771017043/Put/seqid=0 2024-12-09T19:03:38,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741903_1079 (size=12301) 2024-12-09T19:03:38,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771078247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771078251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771078335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,336 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2145 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771078354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771078354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,355 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771078365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,501 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/bdf1966cbcc24a27bce100e8aad5e9c0 2024-12-09T19:03:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/d4b5cbe21b8f4b7ba8e49e146e15c400 is 50, key is test_row_0/B:col10/1733771017043/Put/seqid=0 2024-12-09T19:03:38,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741904_1080 (size=12301) 2024-12-09T19:03:38,514 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/d4b5cbe21b8f4b7ba8e49e146e15c400 2024-12-09T19:03:38,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/749fbb763dd54d218437143d9df28a67 is 50, key is test_row_0/C:col10/1733771017043/Put/seqid=0 2024-12-09T19:03:38,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741905_1081 (size=12301) 2024-12-09T19:03:38,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771078564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771078574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771078875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771078884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:38,926 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/749fbb763dd54d218437143d9df28a67 2024-12-09T19:03:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/bdf1966cbcc24a27bce100e8aad5e9c0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bdf1966cbcc24a27bce100e8aad5e9c0 2024-12-09T19:03:38,939 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bdf1966cbcc24a27bce100e8aad5e9c0, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T19:03:38,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/d4b5cbe21b8f4b7ba8e49e146e15c400 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d4b5cbe21b8f4b7ba8e49e146e15c400 2024-12-09T19:03:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d4b5cbe21b8f4b7ba8e49e146e15c400, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T19:03:38,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/749fbb763dd54d218437143d9df28a67 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/749fbb763dd54d218437143d9df28a67 2024-12-09T19:03:38,955 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/749fbb763dd54d218437143d9df28a67, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T19:03:38,957 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 07169db55e935a0918c76ee552d19727 in 866ms, sequenceid=327, compaction requested=true 2024-12-09T19:03:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T19:03:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T19:03:38,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T19:03:38,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1100 sec 2024-12-09T19:03:38,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1190 sec 2024-12-09T19:03:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T19:03:38,994 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:38,995 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T19:03:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T19:03:38,997 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:38,998 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:38,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T19:03:39,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T19:03:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:39,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-09T19:03:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/353da7ce5d5e435bb90e8b6cf4df2abf is 50, key is test_row_0/A:col10/1733771018226/Put/seqid=0 2024-12-09T19:03:39,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741906_1082 (size=12301) 2024-12-09T19:03:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T19:03:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:39,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771079403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:39,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771079404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771079515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771079515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:39,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/353da7ce5d5e435bb90e8b6cf4df2abf 2024-12-09T19:03:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/5ae28ed711894749aec498077714db2d is 50, key is test_row_0/B:col10/1733771018226/Put/seqid=0 2024-12-09T19:03:39,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741907_1083 (size=12301) 2024-12-09T19:03:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T19:03:39,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771079724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771079725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:39,992 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/5ae28ed711894749aec498077714db2d 2024-12-09T19:03:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2dc2132e2e1e4ee8a0d58e55dd051dd6 is 50, key is test_row_0/C:col10/1733771018226/Put/seqid=0 2024-12-09T19:03:40,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741908_1084 (size=12301) 2024-12-09T19:03:40,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771080034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771080035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T19:03:40,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771080344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,349 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:40,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771080365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,367 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2dc2132e2e1e4ee8a0d58e55dd051dd6 2024-12-09T19:03:40,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/353da7ce5d5e435bb90e8b6cf4df2abf as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/353da7ce5d5e435bb90e8b6cf4df2abf 2024-12-09T19:03:40,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/353da7ce5d5e435bb90e8b6cf4df2abf, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T19:03:40,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/5ae28ed711894749aec498077714db2d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/5ae28ed711894749aec498077714db2d 2024-12-09T19:03:40,427 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/5ae28ed711894749aec498077714db2d, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T19:03:40,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/2dc2132e2e1e4ee8a0d58e55dd051dd6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2dc2132e2e1e4ee8a0d58e55dd051dd6 2024-12-09T19:03:40,433 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2dc2132e2e1e4ee8a0d58e55dd051dd6, entries=150, sequenceid=353, filesize=12.0 K 2024-12-09T19:03:40,434 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 07169db55e935a0918c76ee552d19727 in 1282ms, sequenceid=353, compaction requested=true 2024-12-09T19:03:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:40,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T19:03:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T19:03:40,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T19:03:40,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-12-09T19:03:40,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4440 sec 2024-12-09T19:03:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:40,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:40,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:03:40,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/e87fd634af9b4af3a84fcb5d94d4e158 is 50, key is test_row_0/A:col10/1733771019395/Put/seqid=0 2024-12-09T19:03:40,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741909_1085 (size=14741) 2024-12-09T19:03:40,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771080591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771080592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771080704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771080705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771080914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771080914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:40,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/e87fd634af9b4af3a84fcb5d94d4e158 2024-12-09T19:03:40,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/944918bf88ec4f45bfaf46ad582c4da0 is 50, key is test_row_0/B:col10/1733771019395/Put/seqid=0 2024-12-09T19:03:40,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741910_1086 (size=12301) 2024-12-09T19:03:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55324 deadline: 1733771081034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:41,037 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T19:03:41,145 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:41,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T19:03:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T19:03:41,147 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:41,148 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:41,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771081224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771081224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T19:03:41,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T19:03:41,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:41,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/944918bf88ec4f45bfaf46ad582c4da0 2024-12-09T19:03:41,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/464313a5fff14550bb2d45dfb1fb3bcf is 50, key is test_row_0/C:col10/1733771019395/Put/seqid=0 2024-12-09T19:03:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741911_1087 (size=12301) 2024-12-09T19:03:41,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T19:03:41,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:41,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T19:03:41,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T19:03:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771081734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:41,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771081735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:41,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T19:03:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T19:03:41,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/464313a5fff14550bb2d45dfb1fb3bcf 2024-12-09T19:03:41,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/e87fd634af9b4af3a84fcb5d94d4e158 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/e87fd634af9b4af3a84fcb5d94d4e158 2024-12-09T19:03:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/e87fd634af9b4af3a84fcb5d94d4e158, entries=200, sequenceid=364, filesize=14.4 K 2024-12-09T19:03:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/944918bf88ec4f45bfaf46ad582c4da0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/944918bf88ec4f45bfaf46ad582c4da0 2024-12-09T19:03:41,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/944918bf88ec4f45bfaf46ad582c4da0, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T19:03:41,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/464313a5fff14550bb2d45dfb1fb3bcf as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/464313a5fff14550bb2d45dfb1fb3bcf 2024-12-09T19:03:41,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/464313a5fff14550bb2d45dfb1fb3bcf, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T19:03:41,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 07169db55e935a0918c76ee552d19727 in 1261ms, sequenceid=364, compaction requested=true 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:41,806 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:41,806 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:41,808 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62187 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T19:03:41,808 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64627 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T19:03:41,808 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:41,808 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:41,808 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,808 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,808 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/9ac9a4baa73641b48feda8cd7c8458dc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7f6f13a589c14e5da182e8e8cc13b311, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d4b5cbe21b8f4b7ba8e49e146e15c400, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/5ae28ed711894749aec498077714db2d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/944918bf88ec4f45bfaf46ad582c4da0] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=60.7 K 2024-12-09T19:03:41,808 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/838fcb1293fc4065bb77c9167895a81c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ed4491b801b42d496ae76021a46b4b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bdf1966cbcc24a27bce100e8aad5e9c0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/353da7ce5d5e435bb90e8b6cf4df2abf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/e87fd634af9b4af3a84fcb5d94d4e158] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=63.1 K 2024-12-09T19:03:41,809 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac9a4baa73641b48feda8cd7c8458dc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733771014984 2024-12-09T19:03:41,809 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 838fcb1293fc4065bb77c9167895a81c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733771014984 2024-12-09T19:03:41,809 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6f13a589c14e5da182e8e8cc13b311, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733771016183 2024-12-09T19:03:41,809 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ed4491b801b42d496ae76021a46b4b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733771016183 2024-12-09T19:03:41,809 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b5cbe21b8f4b7ba8e49e146e15c400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733771017036 2024-12-09T19:03:41,809 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdf1966cbcc24a27bce100e8aad5e9c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733771017036 2024-12-09T19:03:41,810 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae28ed711894749aec498077714db2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733771018226 2024-12-09T19:03:41,810 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353da7ce5d5e435bb90e8b6cf4df2abf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733771018226 2024-12-09T19:03:41,810 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 944918bf88ec4f45bfaf46ad582c4da0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733771019395 2024-12-09T19:03:41,810 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87fd634af9b4af3a84fcb5d94d4e158, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733771019395 2024-12-09T19:03:41,821 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:41,821 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:41,821 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/30131a9e7087466fbb19fdeb898923c7 is 50, key is test_row_0/B:col10/1733771019395/Put/seqid=0 2024-12-09T19:03:41,821 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/8670394960a44f738f0e39b17e13f220 is 50, key is test_row_0/A:col10/1733771019395/Put/seqid=0 2024-12-09T19:03:41,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741912_1088 (size=13153) 2024-12-09T19:03:41,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741913_1089 (size=13153) 2024-12-09T19:03:41,845 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/8670394960a44f738f0e39b17e13f220 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/8670394960a44f738f0e39b17e13f220 2024-12-09T19:03:41,845 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/30131a9e7087466fbb19fdeb898923c7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/30131a9e7087466fbb19fdeb898923c7 2024-12-09T19:03:41,851 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 8670394960a44f738f0e39b17e13f220(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:41,851 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 30131a9e7087466fbb19fdeb898923c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:41,851 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:41,851 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:41,851 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=11, startTime=1733771021806; duration=0sec 2024-12-09T19:03:41,851 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=11, startTime=1733771021806; duration=0sec 2024-12-09T19:03:41,852 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:41,852 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:41,852 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:41,852 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:41,852 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T19:03:41,855 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62187 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T19:03:41,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:41,856 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,856 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/8b87dadf97c64becaead04c561f070c5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e450870015784a67b277e4cce289de9e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/749fbb763dd54d218437143d9df28a67, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2dc2132e2e1e4ee8a0d58e55dd051dd6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/464313a5fff14550bb2d45dfb1fb3bcf] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=60.7 K 2024-12-09T19:03:41,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b87dadf97c64becaead04c561f070c5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733771014984 2024-12-09T19:03:41,857 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e450870015784a67b277e4cce289de9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733771016183 2024-12-09T19:03:41,857 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 749fbb763dd54d218437143d9df28a67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733771017036 2024-12-09T19:03:41,858 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dc2132e2e1e4ee8a0d58e55dd051dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733771018226 2024-12-09T19:03:41,858 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 464313a5fff14550bb2d45dfb1fb3bcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733771019395 2024-12-09T19:03:41,873 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:41,873 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/14c4836f58114bce9239f4a338c5afa4 is 50, key is test_row_0/C:col10/1733771019395/Put/seqid=0 2024-12-09T19:03:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741914_1090 (size=13153) 2024-12-09T19:03:41,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T19:03:41,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:41,914 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:03:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/d84aad736e7749b2abe932a29181d7be is 50, key is test_row_0/A:col10/1733771020569/Put/seqid=0 2024-12-09T19:03:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741915_1091 (size=12301) 2024-12-09T19:03:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T19:03:42,285 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/14c4836f58114bce9239f4a338c5afa4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14c4836f58114bce9239f4a338c5afa4 2024-12-09T19:03:42,292 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 14c4836f58114bce9239f4a338c5afa4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:42,292 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:42,292 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=11, startTime=1733771021806; duration=0sec 2024-12-09T19:03:42,292 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:42,292 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:42,325 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/d84aad736e7749b2abe932a29181d7be 2024-12-09T19:03:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/40cebdbae0754c349b95b26cd4c68bb8 is 50, key is test_row_0/B:col10/1733771020569/Put/seqid=0 2024-12-09T19:03:42,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741916_1092 (size=12301) 2024-12-09T19:03:42,340 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/40cebdbae0754c349b95b26cd4c68bb8 2024-12-09T19:03:42,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/250587c783b74bf0beadb8d305a93785 is 50, key is test_row_0/C:col10/1733771020569/Put/seqid=0 2024-12-09T19:03:42,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741917_1093 (size=12301) 2024-12-09T19:03:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:42,753 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/250587c783b74bf0beadb8d305a93785 2024-12-09T19:03:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771082756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771082756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:42,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/d84aad736e7749b2abe932a29181d7be as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d84aad736e7749b2abe932a29181d7be 2024-12-09T19:03:42,766 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d84aad736e7749b2abe932a29181d7be, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T19:03:42,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/40cebdbae0754c349b95b26cd4c68bb8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/40cebdbae0754c349b95b26cd4c68bb8 2024-12-09T19:03:42,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/40cebdbae0754c349b95b26cd4c68bb8, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T19:03:42,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/250587c783b74bf0beadb8d305a93785 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/250587c783b74bf0beadb8d305a93785 2024-12-09T19:03:42,789 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/250587c783b74bf0beadb8d305a93785, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T19:03:42,790 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 07169db55e935a0918c76ee552d19727 in 876ms, sequenceid=391, compaction requested=false 2024-12-09T19:03:42,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:42,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:42,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T19:03:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T19:03:42,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T19:03:42,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-12-09T19:03:42,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6500 sec 2024-12-09T19:03:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:42,867 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:42,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:03:42,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/d96613d1b1b74641be2fa5d25612101a is 50, key is test_row_0/A:col10/1733771022750/Put/seqid=0 2024-12-09T19:03:42,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741918_1094 (size=17181) 2024-12-09T19:03:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771082904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771082906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771083014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771083014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771083224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771083224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/d96613d1b1b74641be2fa5d25612101a 2024-12-09T19:03:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T19:03:43,294 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:43,295 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-09T19:03:43,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T19:03:43,299 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:43,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:43,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/17cac29d50924c4c8a60dea3d1b40e6c is 50, key is test_row_0/B:col10/1733771022750/Put/seqid=0 2024-12-09T19:03:43,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741919_1095 (size=12301) 2024-12-09T19:03:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T19:03:43,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T19:03:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771083534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771083534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:43,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T19:03:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T19:03:43,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/17cac29d50924c4c8a60dea3d1b40e6c 2024-12-09T19:03:43,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/311028b987a547a1a2a865d41d455aff is 50, key is test_row_0/C:col10/1733771022750/Put/seqid=0 2024-12-09T19:03:43,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741920_1096 (size=12301) 2024-12-09T19:03:43,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T19:03:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T19:03:43,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:43,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T19:03:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771084044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771084044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T19:03:44,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:44,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:44,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:44,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:44,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:44,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/311028b987a547a1a2a865d41d455aff 2024-12-09T19:03:44,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/d96613d1b1b74641be2fa5d25612101a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d96613d1b1b74641be2fa5d25612101a 2024-12-09T19:03:44,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d96613d1b1b74641be2fa5d25612101a, entries=250, sequenceid=405, filesize=16.8 K 2024-12-09T19:03:44,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/17cac29d50924c4c8a60dea3d1b40e6c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/17cac29d50924c4c8a60dea3d1b40e6c 2024-12-09T19:03:44,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/17cac29d50924c4c8a60dea3d1b40e6c, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T19:03:44,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/311028b987a547a1a2a865d41d455aff as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/311028b987a547a1a2a865d41d455aff 2024-12-09T19:03:44,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/311028b987a547a1a2a865d41d455aff, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T19:03:44,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 07169db55e935a0918c76ee552d19727 in 1281ms, sequenceid=405, compaction requested=true 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:44,148 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:44,148 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:44,149 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:44,149 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:44,149 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:44,149 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:44,150 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/8670394960a44f738f0e39b17e13f220, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d84aad736e7749b2abe932a29181d7be, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d96613d1b1b74641be2fa5d25612101a] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=41.6 K 2024-12-09T19:03:44,150 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:44,150 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:44,150 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/30131a9e7087466fbb19fdeb898923c7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/40cebdbae0754c349b95b26cd4c68bb8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/17cac29d50924c4c8a60dea3d1b40e6c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=36.9 K 2024-12-09T19:03:44,150 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8670394960a44f738f0e39b17e13f220, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733771019395 2024-12-09T19:03:44,150 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 30131a9e7087466fbb19fdeb898923c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733771019395 2024-12-09T19:03:44,150 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84aad736e7749b2abe932a29181d7be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733771020569 2024-12-09T19:03:44,150 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 40cebdbae0754c349b95b26cd4c68bb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733771020569 2024-12-09T19:03:44,151 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting d96613d1b1b74641be2fa5d25612101a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733771022745 2024-12-09T19:03:44,151 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cac29d50924c4c8a60dea3d1b40e6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733771022750 2024-12-09T19:03:44,158 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:44,158 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/c5eeaebb8adb4f42abab8fc090f628f8 is 50, key is test_row_0/A:col10/1733771022750/Put/seqid=0 2024-12-09T19:03:44,164 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#85 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:44,165 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/3ba352ff5e5c485f91f7a25b050fbf42 is 50, key is test_row_0/B:col10/1733771022750/Put/seqid=0 2024-12-09T19:03:44,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741921_1097 (size=13255) 2024-12-09T19:03:44,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741922_1098 (size=13255) 2024-12-09T19:03:44,210 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/c5eeaebb8adb4f42abab8fc090f628f8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/c5eeaebb8adb4f42abab8fc090f628f8 2024-12-09T19:03:44,211 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/3ba352ff5e5c485f91f7a25b050fbf42 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/3ba352ff5e5c485f91f7a25b050fbf42 2024-12-09T19:03:44,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T19:03:44,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:44,216 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:03:44,219 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into 3ba352ff5e5c485f91f7a25b050fbf42(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:44,219 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:44,220 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=13, startTime=1733771024148; duration=0sec 2024-12-09T19:03:44,220 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into c5eeaebb8adb4f42abab8fc090f628f8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:44,221 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:44,221 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=13, startTime=1733771024148; duration=0sec 2024-12-09T19:03:44,221 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:44,221 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:44,221 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:44,221 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:44,221 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:44,222 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:44,222 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:44,223 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:44,223 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14c4836f58114bce9239f4a338c5afa4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/250587c783b74bf0beadb8d305a93785, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/311028b987a547a1a2a865d41d455aff] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=36.9 K 2024-12-09T19:03:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/044021b7c20245f093ca51b4e7ce25b1 is 50, key is test_row_0/A:col10/1733771022903/Put/seqid=0 2024-12-09T19:03:44,223 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c4836f58114bce9239f4a338c5afa4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733771019395 2024-12-09T19:03:44,225 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 250587c783b74bf0beadb8d305a93785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733771020569 2024-12-09T19:03:44,227 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 311028b987a547a1a2a865d41d455aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733771022750 2024-12-09T19:03:44,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741923_1099 (size=12301) 2024-12-09T19:03:44,238 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/044021b7c20245f093ca51b4e7ce25b1 2024-12-09T19:03:44,241 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:44,242 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/14bc214623f9481ca313e0ec61c24a2f is 50, key is test_row_0/C:col10/1733771022750/Put/seqid=0 2024-12-09T19:03:44,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/240409bb9fbc48668929f92106247c56 is 50, key is test_row_0/B:col10/1733771022903/Put/seqid=0 2024-12-09T19:03:44,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741924_1100 (size=13255) 2024-12-09T19:03:44,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741925_1101 (size=12301) 2024-12-09T19:03:44,260 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/14bc214623f9481ca313e0ec61c24a2f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14bc214623f9481ca313e0ec61c24a2f 2024-12-09T19:03:44,268 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into 14bc214623f9481ca313e0ec61c24a2f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:44,268 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:44,268 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=13, startTime=1733771024148; duration=0sec 2024-12-09T19:03:44,268 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:44,268 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:44,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. as already flushing 2024-12-09T19:03:44,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771084388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771084388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T19:03:44,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771084494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771084494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,657 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/240409bb9fbc48668929f92106247c56 2024-12-09T19:03:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f614fe360a4647f89a3ed5565d61eaae is 50, key is test_row_0/C:col10/1733771022903/Put/seqid=0 2024-12-09T19:03:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741926_1102 (size=12301) 2024-12-09T19:03:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771084705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771084705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:44,917 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-09T19:03:44,917 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-09T19:03:44,917 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:44,917 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:44,919 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-09T19:03:44,919 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:44,921 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,921 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,921 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,921 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,921 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:44,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:44,922 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:44,922 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-09T19:03:44,922 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:44,922 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,923 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:44,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:45,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:45,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1733771085015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55352 deadline: 1733771085015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:45,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:45,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1733771085055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55344 deadline: 1733771085055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:45,056 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:45,057 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 07169db55e935a0918c76ee552d19727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:03:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:03:45,071 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f614fe360a4647f89a3ed5565d61eaae 2024-12-09T19:03:45,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/044021b7c20245f093ca51b4e7ce25b1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/044021b7c20245f093ca51b4e7ce25b1 2024-12-09T19:03:45,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/044021b7c20245f093ca51b4e7ce25b1, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T19:03:45,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/240409bb9fbc48668929f92106247c56 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/240409bb9fbc48668929f92106247c56 2024-12-09T19:03:45,092 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/240409bb9fbc48668929f92106247c56, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T19:03:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/f614fe360a4647f89a3ed5565d61eaae as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f614fe360a4647f89a3ed5565d61eaae 2024-12-09T19:03:45,098 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f614fe360a4647f89a3ed5565d61eaae, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T19:03:45,099 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 07169db55e935a0918c76ee552d19727 in 883ms, sequenceid=428, compaction requested=false 2024-12-09T19:03:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-09T19:03:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-09T19:03:45,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T19:03:45,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-09T19:03:45,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.8070 sec 2024-12-09T19:03:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T19:03:45,445 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:45,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:45,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-09T19:03:45,530 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-09T19:03:45,531 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-09T19:03:45,531 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:45,531 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:45,531 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:45,531 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:45,531 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:45,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:45,532 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:45,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:45,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/04e28925a72f4c6f9485ac34a389125d is 50, key is test_row_0/A:col10/1733771025526/Put/seqid=0 2024-12-09T19:03:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741927_1103 (size=9857) 2024-12-09T19:03:45,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/04e28925a72f4c6f9485ac34a389125d 2024-12-09T19:03:45,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/881493dd3561414b8fea446937988921 is 50, key is test_row_0/B:col10/1733771025526/Put/seqid=0 2024-12-09T19:03:45,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741928_1104 (size=9857) 2024-12-09T19:03:46,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/881493dd3561414b8fea446937988921 2024-12-09T19:03:46,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/47866e52f1554c77b7098ee3f774393b is 50, key is test_row_0/C:col10/1733771025526/Put/seqid=0 2024-12-09T19:03:46,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741929_1105 (size=9857) 2024-12-09T19:03:46,699 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T19:03:46,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/47866e52f1554c77b7098ee3f774393b 2024-12-09T19:03:46,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/04e28925a72f4c6f9485ac34a389125d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/04e28925a72f4c6f9485ac34a389125d 2024-12-09T19:03:46,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/04e28925a72f4c6f9485ac34a389125d, entries=100, sequenceid=446, filesize=9.6 K 2024-12-09T19:03:46,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/881493dd3561414b8fea446937988921 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/881493dd3561414b8fea446937988921 2024-12-09T19:03:46,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/881493dd3561414b8fea446937988921, entries=100, sequenceid=446, filesize=9.6 K 2024-12-09T19:03:46,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/47866e52f1554c77b7098ee3f774393b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/47866e52f1554c77b7098ee3f774393b 2024-12-09T19:03:46,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/47866e52f1554c77b7098ee3f774393b, entries=100, sequenceid=446, filesize=9.6 K 2024-12-09T19:03:46,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 07169db55e935a0918c76ee552d19727 in 1283ms, sequenceid=446, compaction requested=true 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:46,813 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:46,813 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07169db55e935a0918c76ee552d19727:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:46,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:46,815 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:46,815 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:46,815 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/A is initiating minor compaction (all files) 2024-12-09T19:03:46,815 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/B is initiating minor compaction (all files) 2024-12-09T19:03:46,815 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/A in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:46,815 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/B in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:46,815 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/c5eeaebb8adb4f42abab8fc090f628f8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/044021b7c20245f093ca51b4e7ce25b1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/04e28925a72f4c6f9485ac34a389125d] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=34.6 K 2024-12-09T19:03:46,815 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/3ba352ff5e5c485f91f7a25b050fbf42, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/240409bb9fbc48668929f92106247c56, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/881493dd3561414b8fea446937988921] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=34.6 K 2024-12-09T19:03:46,815 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5eeaebb8adb4f42abab8fc090f628f8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733771022750 2024-12-09T19:03:46,815 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba352ff5e5c485f91f7a25b050fbf42, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733771022750 2024-12-09T19:03:46,816 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 044021b7c20245f093ca51b4e7ce25b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733771022898 2024-12-09T19:03:46,816 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 240409bb9fbc48668929f92106247c56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733771022898 2024-12-09T19:03:46,816 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e28925a72f4c6f9485ac34a389125d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733771025526 2024-12-09T19:03:46,816 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 881493dd3561414b8fea446937988921, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733771025526 2024-12-09T19:03:46,826 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:46,826 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:46,826 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/a07eea6edfc14395871556a66f0321c6 is 50, key is test_row_0/B:col10/1733771025526/Put/seqid=0 2024-12-09T19:03:46,826 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/689ddadf81a84d45bd7da2a4a785994f is 50, key is test_row_0/A:col10/1733771025526/Put/seqid=0 2024-12-09T19:03:46,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741931_1107 (size=13357) 2024-12-09T19:03:46,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741930_1106 (size=13357) 2024-12-09T19:03:47,080 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-09T19:03:47,080 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-09T19:03:47,080 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:47,080 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:47,081 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:47,081 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:47,081 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:47,081 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:47,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:47,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:47,247 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/a07eea6edfc14395871556a66f0321c6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a07eea6edfc14395871556a66f0321c6 2024-12-09T19:03:47,247 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/689ddadf81a84d45bd7da2a4a785994f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/689ddadf81a84d45bd7da2a4a785994f 2024-12-09T19:03:47,253 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/B of 07169db55e935a0918c76ee552d19727 into a07eea6edfc14395871556a66f0321c6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:47,253 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/A of 07169db55e935a0918c76ee552d19727 into 689ddadf81a84d45bd7da2a4a785994f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:47,253 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/A, priority=13, startTime=1733771026813; duration=0sec 2024-12-09T19:03:47,253 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/B, priority=13, startTime=1733771026813; duration=0sec 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:A 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:B 2024-12-09T19:03:47,253 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:47,254 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:47,255 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 07169db55e935a0918c76ee552d19727/C is initiating minor compaction (all files) 2024-12-09T19:03:47,255 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07169db55e935a0918c76ee552d19727/C in TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:47,255 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14bc214623f9481ca313e0ec61c24a2f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f614fe360a4647f89a3ed5565d61eaae, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/47866e52f1554c77b7098ee3f774393b] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp, totalSize=34.6 K 2024-12-09T19:03:47,255 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14bc214623f9481ca313e0ec61c24a2f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733771022750 2024-12-09T19:03:47,256 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting f614fe360a4647f89a3ed5565d61eaae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733771022898 2024-12-09T19:03:47,256 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47866e52f1554c77b7098ee3f774393b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733771025526 2024-12-09T19:03:47,263 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07169db55e935a0918c76ee552d19727#C#compaction#95 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:47,264 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/e3c96d13f7a648d394ef89fc25748f58 is 50, key is test_row_0/C:col10/1733771025526/Put/seqid=0 2024-12-09T19:03:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741932_1108 (size=13357) 2024-12-09T19:03:47,677 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/e3c96d13f7a648d394ef89fc25748f58 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e3c96d13f7a648d394ef89fc25748f58 2024-12-09T19:03:47,685 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07169db55e935a0918c76ee552d19727/C of 07169db55e935a0918c76ee552d19727 into e3c96d13f7a648d394ef89fc25748f58(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:47,686 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07169db55e935a0918c76ee552d19727: 2024-12-09T19:03:47,686 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., storeName=07169db55e935a0918c76ee552d19727/C, priority=13, startTime=1733771026813; duration=0sec 2024-12-09T19:03:47,686 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:47,686 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07169db55e935a0918c76ee552d19727:C 2024-12-09T19:03:51,118 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-09T19:03:51,119 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:03:51,119 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:51,119 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:51,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:51,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T19:03:51,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-09T19:03:51,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-09T19:03:51,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-09T19:03:51,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-09T19:03:51,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-09T19:03:51,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T19:03:51,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4840 2024-12-09T19:03:51,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4778 2024-12-09T19:03:51,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T19:03:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2169 2024-12-09T19:03:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6503 rows 2024-12-09T19:03:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2171 2024-12-09T19:03:51,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6508 rows 2024-12-09T19:03:51,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:03:51,123 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:03:51,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:51,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:51,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:03:51,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T19:03:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T19:03:51,134 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771031134"}]},"ts":"1733771031134"} 2024-12-09T19:03:51,136 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T19:03:51,136 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T19:03:51,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:03:51,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, UNASSIGN}] 2024-12-09T19:03:51,143 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, UNASSIGN 2024-12-09T19:03:51,144 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=07169db55e935a0918c76ee552d19727, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:51,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, UNASSIGN because future has completed 2024-12-09T19:03:51,147 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:03:51,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:03:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T19:03:51,306 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:51,307 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:03:51,308 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 07169db55e935a0918c76ee552d19727, disabling compactions & flushes 2024-12-09T19:03:51,308 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:51,308 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:51,308 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. after waiting 0 ms 2024-12-09T19:03:51,308 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:51,309 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 07169db55e935a0918c76ee552d19727 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-12-09T19:03:51,315 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/378e726eec264351b500aa4b95ee51ea is 50, key is test_row_1/A:col10/1733771031115/Put/seqid=0 2024-12-09T19:03:51,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741933_1109 (size=9857) 2024-12-09T19:03:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T19:03:51,722 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/378e726eec264351b500aa4b95ee51ea 2024-12-09T19:03:51,740 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/d547e8ddc4ba47f2b433f5b800d23d8f is 50, key is test_row_1/B:col10/1733771031115/Put/seqid=0 2024-12-09T19:03:51,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741934_1110 (size=9857) 2024-12-09T19:03:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T19:03:52,146 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/d547e8ddc4ba47f2b433f5b800d23d8f 2024-12-09T19:03:52,160 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/bbe484b20790469795491a36e4abf145 is 50, key is test_row_1/C:col10/1733771031115/Put/seqid=0 2024-12-09T19:03:52,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741935_1111 (size=9857) 2024-12-09T19:03:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T19:03:52,566 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/bbe484b20790469795491a36e4abf145 2024-12-09T19:03:52,578 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/A/378e726eec264351b500aa4b95ee51ea as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/378e726eec264351b500aa4b95ee51ea 2024-12-09T19:03:52,584 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/378e726eec264351b500aa4b95ee51ea, entries=100, sequenceid=455, filesize=9.6 K 2024-12-09T19:03:52,585 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/B/d547e8ddc4ba47f2b433f5b800d23d8f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d547e8ddc4ba47f2b433f5b800d23d8f 2024-12-09T19:03:52,591 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d547e8ddc4ba47f2b433f5b800d23d8f, entries=100, sequenceid=455, filesize=9.6 K 2024-12-09T19:03:52,592 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/.tmp/C/bbe484b20790469795491a36e4abf145 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/bbe484b20790469795491a36e4abf145 2024-12-09T19:03:52,597 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/bbe484b20790469795491a36e4abf145, entries=100, sequenceid=455, filesize=9.6 K 2024-12-09T19:03:52,598 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 07169db55e935a0918c76ee552d19727 in 1290ms, sequenceid=455, compaction requested=false 2024-12-09T19:03:52,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/0956ea5dd99c4f1a8334c352c5105ad0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/012af5e7932a4cd49927a80a79f1b84a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/7551ddc261c64959972e429e8180c223, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ef76fbacd6b484e898b034cef2e83e3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b34f022e720d44618d30845e69cdbd60, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/472f2f244e8a41b68f10320ac6bd1795, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/28cce9a265b644d78fe8b94f6b5d2396, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b054e3bf524f485db4cfeeb671b319c8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/048b7f4d523a45968e34edea5002ee34, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/44a336afdf5c44fe8907eca84e03a319, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/9c703130c3f9434f951f50a244716d1d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/6a807e84fb5146d9b814cdbbbe18cc71, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3de66cfbc65440ffb5ccf9fee6a6650c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a3d3c5561cf64d3ea67b22f618e022cd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a423763158394855a895bd5274f4f559, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/49a8c8bab9144a0faafcfa17b36cf04f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/1b7d226672ae4c00bfcdc4629dc9fa07, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3122bce69c8f48659f6696d147b3b843, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/838fcb1293fc4065bb77c9167895a81c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/98ac836212744f98a52a83f487db221c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ed4491b801b42d496ae76021a46b4b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bdf1966cbcc24a27bce100e8aad5e9c0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/353da7ce5d5e435bb90e8b6cf4df2abf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/e87fd634af9b4af3a84fcb5d94d4e158, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/8670394960a44f738f0e39b17e13f220, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d84aad736e7749b2abe932a29181d7be, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d96613d1b1b74641be2fa5d25612101a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/c5eeaebb8adb4f42abab8fc090f628f8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/044021b7c20245f093ca51b4e7ce25b1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/04e28925a72f4c6f9485ac34a389125d] to archive 2024-12-09T19:03:52,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:03:52,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/0956ea5dd99c4f1a8334c352c5105ad0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/0956ea5dd99c4f1a8334c352c5105ad0 2024-12-09T19:03:52,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/012af5e7932a4cd49927a80a79f1b84a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/012af5e7932a4cd49927a80a79f1b84a 2024-12-09T19:03:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/7551ddc261c64959972e429e8180c223 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/7551ddc261c64959972e429e8180c223 2024-12-09T19:03:52,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ef76fbacd6b484e898b034cef2e83e3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ef76fbacd6b484e898b034cef2e83e3 2024-12-09T19:03:52,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b34f022e720d44618d30845e69cdbd60 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b34f022e720d44618d30845e69cdbd60 2024-12-09T19:03:52,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/472f2f244e8a41b68f10320ac6bd1795 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/472f2f244e8a41b68f10320ac6bd1795 2024-12-09T19:03:52,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/28cce9a265b644d78fe8b94f6b5d2396 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/28cce9a265b644d78fe8b94f6b5d2396 2024-12-09T19:03:52,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b054e3bf524f485db4cfeeb671b319c8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/b054e3bf524f485db4cfeeb671b319c8 2024-12-09T19:03:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bcd1e8304e5e43cd9f5ce4f261ff8eb4 2024-12-09T19:03:52,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/048b7f4d523a45968e34edea5002ee34 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/048b7f4d523a45968e34edea5002ee34 2024-12-09T19:03:52,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/44a336afdf5c44fe8907eca84e03a319 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/44a336afdf5c44fe8907eca84e03a319 2024-12-09T19:03:52,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/9c703130c3f9434f951f50a244716d1d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/9c703130c3f9434f951f50a244716d1d 2024-12-09T19:03:52,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/6a807e84fb5146d9b814cdbbbe18cc71 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/6a807e84fb5146d9b814cdbbbe18cc71 2024-12-09T19:03:52,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3de66cfbc65440ffb5ccf9fee6a6650c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3de66cfbc65440ffb5ccf9fee6a6650c 2024-12-09T19:03:52,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a3d3c5561cf64d3ea67b22f618e022cd to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a3d3c5561cf64d3ea67b22f618e022cd 2024-12-09T19:03:52,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a423763158394855a895bd5274f4f559 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/a423763158394855a895bd5274f4f559 2024-12-09T19:03:52,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/49a8c8bab9144a0faafcfa17b36cf04f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/49a8c8bab9144a0faafcfa17b36cf04f 2024-12-09T19:03:52,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/1b7d226672ae4c00bfcdc4629dc9fa07 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/1b7d226672ae4c00bfcdc4629dc9fa07 2024-12-09T19:03:52,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3122bce69c8f48659f6696d147b3b843 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3122bce69c8f48659f6696d147b3b843 2024-12-09T19:03:52,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/838fcb1293fc4065bb77c9167895a81c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/838fcb1293fc4065bb77c9167895a81c 2024-12-09T19:03:52,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/98ac836212744f98a52a83f487db221c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/98ac836212744f98a52a83f487db221c 2024-12-09T19:03:52,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ed4491b801b42d496ae76021a46b4b4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/3ed4491b801b42d496ae76021a46b4b4 2024-12-09T19:03:52,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bdf1966cbcc24a27bce100e8aad5e9c0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/bdf1966cbcc24a27bce100e8aad5e9c0 2024-12-09T19:03:52,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/353da7ce5d5e435bb90e8b6cf4df2abf to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/353da7ce5d5e435bb90e8b6cf4df2abf 2024-12-09T19:03:52,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/e87fd634af9b4af3a84fcb5d94d4e158 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/e87fd634af9b4af3a84fcb5d94d4e158 2024-12-09T19:03:52,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/8670394960a44f738f0e39b17e13f220 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/8670394960a44f738f0e39b17e13f220 2024-12-09T19:03:52,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d84aad736e7749b2abe932a29181d7be to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d84aad736e7749b2abe932a29181d7be 2024-12-09T19:03:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d96613d1b1b74641be2fa5d25612101a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/d96613d1b1b74641be2fa5d25612101a 2024-12-09T19:03:52,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/c5eeaebb8adb4f42abab8fc090f628f8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/c5eeaebb8adb4f42abab8fc090f628f8 2024-12-09T19:03:52,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/044021b7c20245f093ca51b4e7ce25b1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/044021b7c20245f093ca51b4e7ce25b1 2024-12-09T19:03:52,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/04e28925a72f4c6f9485ac34a389125d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/04e28925a72f4c6f9485ac34a389125d 2024-12-09T19:03:52,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/0f8b0063b1e14a2597cf4fe581e9e121, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a45c46e015e541248059de01ceffbc6e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/74a665effd344a0997f903be8f54f3f9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8c721fd7c63740979d5aabe3e4722318, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/13971cdad854490ea6b9b73d3563dfd6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/778be0b8341b472d8dc192f0fb20aafa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/743caed0b8c0484b99cd85cc0bd93f9b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/c322bb84102241e19e3713c073d10af6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/21ef5d6acb1c4098800c2bbb0d157fc0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8d0cc1a259674e86990405db3391beb4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54cfc74fd33147d1afc787da15c07d14, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/091755124fcb4716bb7e0a3ac7ff817c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/41561b2ff10c454cb0e73bac673a54cd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/df54bf1c7acb4604bec78a1c4417e764, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/f31545e237634c378abaf8f60fea54b0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/56f82e1e5d6241adac0371caf8181bd5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7626d34474f240f3addbcf8e8d417720, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/270d4dc8c97447dc9f3e6ae0a72c6ee8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/fc4fad72368d47b7be6f0fa752cb972f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/9ac9a4baa73641b48feda8cd7c8458dc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54b8b4a2cdf14904a58d2d9a1dbda20f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7f6f13a589c14e5da182e8e8cc13b311, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d4b5cbe21b8f4b7ba8e49e146e15c400, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/5ae28ed711894749aec498077714db2d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/30131a9e7087466fbb19fdeb898923c7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/944918bf88ec4f45bfaf46ad582c4da0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/40cebdbae0754c349b95b26cd4c68bb8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/3ba352ff5e5c485f91f7a25b050fbf42, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/17cac29d50924c4c8a60dea3d1b40e6c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/240409bb9fbc48668929f92106247c56, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/881493dd3561414b8fea446937988921] to archive 2024-12-09T19:03:52,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:03:52,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/0f8b0063b1e14a2597cf4fe581e9e121 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/0f8b0063b1e14a2597cf4fe581e9e121 2024-12-09T19:03:52,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a45c46e015e541248059de01ceffbc6e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a45c46e015e541248059de01ceffbc6e 2024-12-09T19:03:52,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/74a665effd344a0997f903be8f54f3f9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/74a665effd344a0997f903be8f54f3f9 2024-12-09T19:03:52,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8c721fd7c63740979d5aabe3e4722318 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8c721fd7c63740979d5aabe3e4722318 2024-12-09T19:03:52,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/13971cdad854490ea6b9b73d3563dfd6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/13971cdad854490ea6b9b73d3563dfd6 2024-12-09T19:03:52,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/778be0b8341b472d8dc192f0fb20aafa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/778be0b8341b472d8dc192f0fb20aafa 2024-12-09T19:03:52,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/743caed0b8c0484b99cd85cc0bd93f9b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/743caed0b8c0484b99cd85cc0bd93f9b 2024-12-09T19:03:52,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/c322bb84102241e19e3713c073d10af6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/c322bb84102241e19e3713c073d10af6 2024-12-09T19:03:52,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/21ef5d6acb1c4098800c2bbb0d157fc0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/21ef5d6acb1c4098800c2bbb0d157fc0 2024-12-09T19:03:52,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8d0cc1a259674e86990405db3391beb4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/8d0cc1a259674e86990405db3391beb4 2024-12-09T19:03:52,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54cfc74fd33147d1afc787da15c07d14 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54cfc74fd33147d1afc787da15c07d14 2024-12-09T19:03:52,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/091755124fcb4716bb7e0a3ac7ff817c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/091755124fcb4716bb7e0a3ac7ff817c 2024-12-09T19:03:52,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/41561b2ff10c454cb0e73bac673a54cd to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/41561b2ff10c454cb0e73bac673a54cd 2024-12-09T19:03:52,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/df54bf1c7acb4604bec78a1c4417e764 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/df54bf1c7acb4604bec78a1c4417e764 2024-12-09T19:03:52,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/f31545e237634c378abaf8f60fea54b0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/f31545e237634c378abaf8f60fea54b0 2024-12-09T19:03:52,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/56f82e1e5d6241adac0371caf8181bd5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/56f82e1e5d6241adac0371caf8181bd5 2024-12-09T19:03:52,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7626d34474f240f3addbcf8e8d417720 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7626d34474f240f3addbcf8e8d417720 2024-12-09T19:03:52,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/270d4dc8c97447dc9f3e6ae0a72c6ee8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/270d4dc8c97447dc9f3e6ae0a72c6ee8 2024-12-09T19:03:52,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/fc4fad72368d47b7be6f0fa752cb972f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/fc4fad72368d47b7be6f0fa752cb972f 2024-12-09T19:03:52,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/9ac9a4baa73641b48feda8cd7c8458dc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/9ac9a4baa73641b48feda8cd7c8458dc 2024-12-09T19:03:52,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54b8b4a2cdf14904a58d2d9a1dbda20f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/54b8b4a2cdf14904a58d2d9a1dbda20f 2024-12-09T19:03:52,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7f6f13a589c14e5da182e8e8cc13b311 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/7f6f13a589c14e5da182e8e8cc13b311 2024-12-09T19:03:52,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d4b5cbe21b8f4b7ba8e49e146e15c400 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d4b5cbe21b8f4b7ba8e49e146e15c400 2024-12-09T19:03:52,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/5ae28ed711894749aec498077714db2d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/5ae28ed711894749aec498077714db2d 2024-12-09T19:03:52,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/30131a9e7087466fbb19fdeb898923c7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/30131a9e7087466fbb19fdeb898923c7 2024-12-09T19:03:52,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/944918bf88ec4f45bfaf46ad582c4da0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/944918bf88ec4f45bfaf46ad582c4da0 2024-12-09T19:03:52,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/40cebdbae0754c349b95b26cd4c68bb8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/40cebdbae0754c349b95b26cd4c68bb8 2024-12-09T19:03:52,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/3ba352ff5e5c485f91f7a25b050fbf42 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/3ba352ff5e5c485f91f7a25b050fbf42 2024-12-09T19:03:52,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/17cac29d50924c4c8a60dea3d1b40e6c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/17cac29d50924c4c8a60dea3d1b40e6c 2024-12-09T19:03:52,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/240409bb9fbc48668929f92106247c56 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/240409bb9fbc48668929f92106247c56 2024-12-09T19:03:52,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/881493dd3561414b8fea446937988921 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/881493dd3561414b8fea446937988921 2024-12-09T19:03:52,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38d2739a147f4c8086abe300e2607144, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/66cb620be9b0468db60dc07a146cd803, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/919277c913e14084a64b0e07eae31bb2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2520baacc0ea44949be49cfd9c7dce25, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2addb27368014b8fbce37bcf4ba71e5b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/ce64e56f92d444c8ad1a2c75fc350386, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc1dff43e16e4ac2b176bf5a0476314f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/4e00994d2de64ba8946162acde62e2f4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5c6760878402478aac33b2868fbc2810, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc546104223f40549a64f515fd27effa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5ce8cf1aa8ce46e39527f4e5e5987d77, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38b09c3e04e3415ab09107485ea93a24, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/821b845d906346e4b3c47f9851d47434, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/52079ba639f84ff7b5bd9a75f883fa93, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5b5686023284438cb15175f2fcf0b123, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2bf31a9ae5d347e1955fcff26c9c5114, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/a2e8db48820c4f5892dde9ad791294f2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/8b87dadf97c64becaead04c561f070c5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f564afdaafca42abacde3f244b10e3fe, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e450870015784a67b277e4cce289de9e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/749fbb763dd54d218437143d9df28a67, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2dc2132e2e1e4ee8a0d58e55dd051dd6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14c4836f58114bce9239f4a338c5afa4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/464313a5fff14550bb2d45dfb1fb3bcf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/250587c783b74bf0beadb8d305a93785, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14bc214623f9481ca313e0ec61c24a2f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/311028b987a547a1a2a865d41d455aff, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f614fe360a4647f89a3ed5565d61eaae, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/47866e52f1554c77b7098ee3f774393b] to archive 2024-12-09T19:03:52,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:03:52,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/9ccf78e2cd9e48b193bfb34fc0e1b9a7 2024-12-09T19:03:52,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f0af30f5a7844bd5b0a5cf4bc4a3c0eb 2024-12-09T19:03:52,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38d2739a147f4c8086abe300e2607144 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38d2739a147f4c8086abe300e2607144 2024-12-09T19:03:52,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/66cb620be9b0468db60dc07a146cd803 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/66cb620be9b0468db60dc07a146cd803 2024-12-09T19:03:52,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/919277c913e14084a64b0e07eae31bb2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/919277c913e14084a64b0e07eae31bb2 2024-12-09T19:03:52,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2520baacc0ea44949be49cfd9c7dce25 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2520baacc0ea44949be49cfd9c7dce25 2024-12-09T19:03:52,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2addb27368014b8fbce37bcf4ba71e5b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2addb27368014b8fbce37bcf4ba71e5b 2024-12-09T19:03:52,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/ce64e56f92d444c8ad1a2c75fc350386 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/ce64e56f92d444c8ad1a2c75fc350386 2024-12-09T19:03:52,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc1dff43e16e4ac2b176bf5a0476314f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc1dff43e16e4ac2b176bf5a0476314f 2024-12-09T19:03:52,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/4e00994d2de64ba8946162acde62e2f4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/4e00994d2de64ba8946162acde62e2f4 2024-12-09T19:03:52,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5c6760878402478aac33b2868fbc2810 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5c6760878402478aac33b2868fbc2810 2024-12-09T19:03:52,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc546104223f40549a64f515fd27effa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/dc546104223f40549a64f515fd27effa 2024-12-09T19:03:52,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5ce8cf1aa8ce46e39527f4e5e5987d77 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5ce8cf1aa8ce46e39527f4e5e5987d77 2024-12-09T19:03:52,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38b09c3e04e3415ab09107485ea93a24 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/38b09c3e04e3415ab09107485ea93a24 2024-12-09T19:03:52,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/821b845d906346e4b3c47f9851d47434 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/821b845d906346e4b3c47f9851d47434 2024-12-09T19:03:52,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/52079ba639f84ff7b5bd9a75f883fa93 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/52079ba639f84ff7b5bd9a75f883fa93 2024-12-09T19:03:52,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5b5686023284438cb15175f2fcf0b123 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/5b5686023284438cb15175f2fcf0b123 2024-12-09T19:03:52,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2bf31a9ae5d347e1955fcff26c9c5114 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2bf31a9ae5d347e1955fcff26c9c5114 2024-12-09T19:03:52,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/a2e8db48820c4f5892dde9ad791294f2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/a2e8db48820c4f5892dde9ad791294f2 2024-12-09T19:03:52,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/8b87dadf97c64becaead04c561f070c5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/8b87dadf97c64becaead04c561f070c5 2024-12-09T19:03:52,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f564afdaafca42abacde3f244b10e3fe to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f564afdaafca42abacde3f244b10e3fe 2024-12-09T19:03:52,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e450870015784a67b277e4cce289de9e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e450870015784a67b277e4cce289de9e 2024-12-09T19:03:52,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/749fbb763dd54d218437143d9df28a67 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/749fbb763dd54d218437143d9df28a67 2024-12-09T19:03:52,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2dc2132e2e1e4ee8a0d58e55dd051dd6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/2dc2132e2e1e4ee8a0d58e55dd051dd6 2024-12-09T19:03:52,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14c4836f58114bce9239f4a338c5afa4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14c4836f58114bce9239f4a338c5afa4 2024-12-09T19:03:52,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/464313a5fff14550bb2d45dfb1fb3bcf to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/464313a5fff14550bb2d45dfb1fb3bcf 2024-12-09T19:03:52,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/250587c783b74bf0beadb8d305a93785 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/250587c783b74bf0beadb8d305a93785 2024-12-09T19:03:52,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14bc214623f9481ca313e0ec61c24a2f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/14bc214623f9481ca313e0ec61c24a2f 2024-12-09T19:03:52,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/311028b987a547a1a2a865d41d455aff to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/311028b987a547a1a2a865d41d455aff 2024-12-09T19:03:52,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f614fe360a4647f89a3ed5565d61eaae to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/f614fe360a4647f89a3ed5565d61eaae 2024-12-09T19:03:52,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/47866e52f1554c77b7098ee3f774393b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/47866e52f1554c77b7098ee3f774393b 2024-12-09T19:03:52,728 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/recovered.edits/458.seqid, newMaxSeqId=458, maxSeqId=1 2024-12-09T19:03:52,729 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. 2024-12-09T19:03:52,729 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 07169db55e935a0918c76ee552d19727: Waiting for close lock at 1733771031308Running coprocessor pre-close hooks at 1733771031308Disabling compacts and flushes for region at 1733771031308Disabling writes for close at 1733771031308Obtaining lock to block concurrent updates at 1733771031309 (+1 ms)Preparing flush snapshotting stores in 07169db55e935a0918c76ee552d19727 at 1733771031309Finished memstore snapshotting TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1733771031309Flushing stores of TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727. at 1733771031310 (+1 ms)Flushing 07169db55e935a0918c76ee552d19727/A: creating writer at 1733771031310Flushing 07169db55e935a0918c76ee552d19727/A: appending metadata at 1733771031314 (+4 ms)Flushing 07169db55e935a0918c76ee552d19727/A: closing flushed file at 1733771031314Flushing 07169db55e935a0918c76ee552d19727/B: creating writer at 1733771031735 (+421 ms)Flushing 07169db55e935a0918c76ee552d19727/B: appending metadata at 1733771031739 (+4 ms)Flushing 07169db55e935a0918c76ee552d19727/B: closing flushed file at 1733771031739Flushing 07169db55e935a0918c76ee552d19727/C: creating writer at 1733771032157 (+418 ms)Flushing 07169db55e935a0918c76ee552d19727/C: appending metadata at 1733771032160 (+3 ms)Flushing 07169db55e935a0918c76ee552d19727/C: closing flushed file at 1733771032160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d6cd46a: reopening flushed file at 1733771032577 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a653264: reopening flushed file at 1733771032584 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68cc04e7: reopening flushed file at 1733771032591 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 07169db55e935a0918c76ee552d19727 in 1290ms, sequenceid=455, compaction requested=false at 1733771032598 (+7 ms)Writing region close event to WAL at 1733771032724 (+126 ms)Running coprocessor post-close hooks at 1733771032729 (+5 ms)Closed at 1733771032729 2024-12-09T19:03:52,732 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 07169db55e935a0918c76ee552d19727 2024-12-09T19:03:52,732 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=07169db55e935a0918c76ee552d19727, regionState=CLOSED 2024-12-09T19:03:52,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:03:52,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T19:03:52,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 07169db55e935a0918c76ee552d19727, server=a437f8b9ba7d,46189,1733770998237 in 1.5880 sec 2024-12-09T19:03:52,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-09T19:03:52,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07169db55e935a0918c76ee552d19727, UNASSIGN in 1.5950 sec 2024-12-09T19:03:52,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T19:03:52,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6010 sec 2024-12-09T19:03:52,742 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771032742"}]},"ts":"1733771032742"} 2024-12-09T19:03:52,743 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T19:03:52,743 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T19:03:52,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6150 sec 2024-12-09T19:03:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T19:03:53,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:53,295 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T19:03:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,302 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T19:03:53,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,307 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727 2024-12-09T19:03:53,311 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/recovered.edits] 2024-12-09T19:03:53,315 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/378e726eec264351b500aa4b95ee51ea to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/378e726eec264351b500aa4b95ee51ea 2024-12-09T19:03:53,317 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/689ddadf81a84d45bd7da2a4a785994f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/A/689ddadf81a84d45bd7da2a4a785994f 2024-12-09T19:03:53,320 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a07eea6edfc14395871556a66f0321c6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/a07eea6edfc14395871556a66f0321c6 2024-12-09T19:03:53,322 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d547e8ddc4ba47f2b433f5b800d23d8f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/B/d547e8ddc4ba47f2b433f5b800d23d8f 2024-12-09T19:03:53,325 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/bbe484b20790469795491a36e4abf145 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/bbe484b20790469795491a36e4abf145 2024-12-09T19:03:53,327 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e3c96d13f7a648d394ef89fc25748f58 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/C/e3c96d13f7a648d394ef89fc25748f58 2024-12-09T19:03:53,329 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/recovered.edits/458.seqid to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727/recovered.edits/458.seqid 2024-12-09T19:03:53,330 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/07169db55e935a0918c76ee552d19727 2024-12-09T19:03:53,330 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T19:03:53,335 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T19:03:53,344 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T19:03:53,367 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T19:03:53,369 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,369 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T19:03:53,369 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733771033369"}]},"ts":"9223372036854775807"} 2024-12-09T19:03:53,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T19:03:53,373 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 07169db55e935a0918c76ee552d19727, NAME => 'TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T19:03:53,373 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T19:03:53,373 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733771033373"}]},"ts":"9223372036854775807"} 2024-12-09T19:03:53,376 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T19:03:53,377 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-12-09T19:03:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T19:03:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T19:03:53,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:53,433 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a437f8b9ba7d:46189-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1672360030_22 at /127.0.0.1:45814 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=225 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2342 (was 2853) 2024-12-09T19:03:53,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=225, ProcessCount=11, AvailableMemoryMB=2341 2024-12-09T19:03:53,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:03:53,443 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:03:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:53,445 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T19:03:53,445 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:53,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-09T19:03:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-09T19:03:53,446 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T19:03:53,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741936_1112 (size=959) 2024-12-09T19:03:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-09T19:03:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-09T19:03:53,859 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:03:53,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741937_1113 (size=53) 2024-12-09T19:03:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-09T19:03:54,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:54,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 524e93dcb7c45b6722e1741895f65463, disabling compactions & flushes 2024-12-09T19:03:54,272 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. after waiting 0 ms 2024-12-09T19:03:54,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,272 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 524e93dcb7c45b6722e1741895f65463: Waiting for close lock at 1733771034272Disabling compacts and flushes for region at 1733771034272Disabling writes for close at 1733771034272Writing region close event to WAL at 1733771034272Closed at 1733771034272 2024-12-09T19:03:54,275 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T19:03:54,275 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733771034275"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733771034275"}]},"ts":"1733771034275"} 2024-12-09T19:03:54,278 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T19:03:54,279 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T19:03:54,279 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771034279"}]},"ts":"1733771034279"} 2024-12-09T19:03:54,281 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T19:03:54,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, ASSIGN}] 2024-12-09T19:03:54,283 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, ASSIGN 2024-12-09T19:03:54,285 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=false 2024-12-09T19:03:54,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:54,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, ASSIGN because future has completed 2024-12-09T19:03:54,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:03:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-09T19:03:54,625 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,626 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:03:54,626 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,626 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:54,627 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,627 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,629 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,633 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524e93dcb7c45b6722e1741895f65463 columnFamilyName A 2024-12-09T19:03:54,633 DEBUG [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:54,634 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(327): Store=524e93dcb7c45b6722e1741895f65463/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:54,634 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,636 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524e93dcb7c45b6722e1741895f65463 columnFamilyName B 2024-12-09T19:03:54,637 DEBUG [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:54,637 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(327): Store=524e93dcb7c45b6722e1741895f65463/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:54,638 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,639 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524e93dcb7c45b6722e1741895f65463 columnFamilyName C 2024-12-09T19:03:54,639 DEBUG [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:54,640 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(327): Store=524e93dcb7c45b6722e1741895f65463/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:54,640 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,641 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,641 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,642 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,642 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,643 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:03:54,644 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,646 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:03:54,646 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 524e93dcb7c45b6722e1741895f65463; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73956863, jitterRate=0.10204313695430756}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:03:54,646 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:54,647 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 524e93dcb7c45b6722e1741895f65463: Running coprocessor pre-open hook at 1733771034627Writing region info on filesystem at 1733771034627Initializing all the Stores at 1733771034628 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771034628Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771034628Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771034628Cleaning up temporary data from old regions at 1733771034643 (+15 ms)Running coprocessor post-open hooks at 1733771034646 (+3 ms)Region opened successfully at 1733771034647 (+1 ms) 2024-12-09T19:03:54,648 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., pid=34, masterSystemTime=1733771034617 2024-12-09T19:03:54,649 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,649 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:54,650 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:54,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:03:54,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-09T19:03:54,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 in 212 msec 2024-12-09T19:03:54,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-09T19:03:54,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, ASSIGN in 372 msec 2024-12-09T19:03:54,657 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T19:03:54,657 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771034657"}]},"ts":"1733771034657"} 2024-12-09T19:03:54,659 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T19:03:54,660 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T19:03:54,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2170 sec 2024-12-09T19:03:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-09T19:03:55,607 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:55,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:55,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:55,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:55,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b44f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:55,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666e63ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:55,618 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:55,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:55,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:03:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:03:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6efa9614 2024-12-09T19:03:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:03:55,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:03:55,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:03:55,628 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:03:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T19:03:55,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741938_1114 (size=995) 2024-12-09T19:03:56,050 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-09T19:03:56,050 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-09T19:03:56,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:03:56,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, REOPEN/MOVE}] 2024-12-09T19:03:56,065 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, REOPEN/MOVE 2024-12-09T19:03:56,066 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, REOPEN/MOVE because future has completed 2024-12-09T19:03:56,068 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:03:56,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:03:56,221 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,221 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:03:56,221 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 524e93dcb7c45b6722e1741895f65463, disabling compactions & flushes 2024-12-09T19:03:56,221 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,221 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,221 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. after waiting 0 ms 2024-12-09T19:03:56,221 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,229 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T19:03:56,230 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,230 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 524e93dcb7c45b6722e1741895f65463: Waiting for close lock at 1733771036221Running coprocessor pre-close hooks at 1733771036221Disabling compacts and flushes for region at 1733771036221Disabling writes for close at 1733771036221Writing region close event to WAL at 1733771036222 (+1 ms)Running coprocessor post-close hooks at 1733771036230 (+8 ms)Closed at 1733771036230 2024-12-09T19:03:56,230 WARN [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 524e93dcb7c45b6722e1741895f65463 to self. 2024-12-09T19:03:56,232 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=CLOSED 2024-12-09T19:03:56,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:03:56,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-09T19:03:56,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 in 168 msec 2024-12-09T19:03:56,240 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, REOPEN/MOVE; state=CLOSED, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=true 2024-12-09T19:03:56,391 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, REOPEN/MOVE because future has completed 2024-12-09T19:03:56,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:03:56,553 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,553 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:03:56,554 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,554 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:03:56,554 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,554 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,560 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,568 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524e93dcb7c45b6722e1741895f65463 columnFamilyName A 2024-12-09T19:03:56,569 DEBUG [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:56,570 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(327): Store=524e93dcb7c45b6722e1741895f65463/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:56,570 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,571 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524e93dcb7c45b6722e1741895f65463 columnFamilyName B 2024-12-09T19:03:56,571 DEBUG [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:56,572 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(327): Store=524e93dcb7c45b6722e1741895f65463/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:56,572 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,572 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 524e93dcb7c45b6722e1741895f65463 columnFamilyName C 2024-12-09T19:03:56,572 DEBUG [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:56,573 INFO [StoreOpener-524e93dcb7c45b6722e1741895f65463-1 {}] regionserver.HStore(327): Store=524e93dcb7c45b6722e1741895f65463/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:03:56,573 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,573 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,574 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,575 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,575 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,576 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:03:56,577 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,577 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 524e93dcb7c45b6722e1741895f65463; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63231595, jitterRate=-0.05777581036090851}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:03:56,577 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,579 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 524e93dcb7c45b6722e1741895f65463: Running coprocessor pre-open hook at 1733771036554Writing region info on filesystem at 1733771036554Initializing all the Stores at 1733771036555 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771036555Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771036556 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771036556Cleaning up temporary data from old regions at 1733771036575 (+19 ms)Running coprocessor post-open hooks at 1733771036577 (+2 ms)Region opened successfully at 1733771036579 (+2 ms) 2024-12-09T19:03:56,580 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., pid=39, masterSystemTime=1733771036549 2024-12-09T19:03:56,582 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,582 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,582 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=OPEN, openSeqNum=5, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:03:56,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-09T19:03:56,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 in 189 msec 2024-12-09T19:03:56,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-09T19:03:56,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, REOPEN/MOVE in 522 msec 2024-12-09T19:03:56,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T19:03:56,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-12-09T19:03:56,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-12-09T19:03:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-09T19:03:56,596 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:56,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,598 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9d3e39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,600 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5b1922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,602 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,603 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2047b0a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,605 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b676b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,607 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b091536, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d4e29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,610 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2667302f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,612 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4240b451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,614 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363ec9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,616 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7303ba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c754b70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e5599e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,623 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ce98f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72257f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:03:56,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:03:56,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:03:56,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:03:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:03:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46866424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:03:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:03:56,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:03:56,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:03:56,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ecc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:03:56,630 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,630 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:56,630 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,630 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,630 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,631 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-09T19:03:56,633 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,633 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,633 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,633 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,634 DEBUG [Thread-595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:03:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T19:03:56,634 DEBUG [Thread-604 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,634 DEBUG [Thread-604 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,635 DEBUG [Thread-589 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,637 DEBUG [Thread-589 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:03:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:03:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,640 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:56,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:56,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:03:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:03:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,652 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:56,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:03:56,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d7c6f95ef0b489da7abf8c1c455566d_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771036648/Put/seqid=0 2024-12-09T19:03:56,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771096700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771096701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771096703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771096705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771096707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741939_1115 (size=14594) 2024-12-09T19:03:56,725 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:56,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d7c6f95ef0b489da7abf8c1c455566d_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d7c6f95ef0b489da7abf8c1c455566d_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/fbb7706248a9402f844a60e96f0305fc, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:56,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/fbb7706248a9402f844a60e96f0305fc is 175, key is test_row_0/A:col10/1733771036648/Put/seqid=0 2024-12-09T19:03:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T19:03:56,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741940_1116 (size=39549) 2024-12-09T19:03:56,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/fbb7706248a9402f844a60e96f0305fc 2024-12-09T19:03:56,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/c1781e3c31d741b59b3af791fe805c67 is 50, key is test_row_0/B:col10/1733771036648/Put/seqid=0 2024-12-09T19:03:56,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T19:03:56,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:56,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:56,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:56,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741941_1117 (size=12001) 2024-12-09T19:03:56,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/c1781e3c31d741b59b3af791fe805c67 2024-12-09T19:03:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771096815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771096815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771096815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771096825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771096825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:56,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/b2ee67c7a1004e379a8a7ef9db712814 is 50, key is test_row_0/C:col10/1733771036648/Put/seqid=0 2024-12-09T19:03:56,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741942_1118 (size=12001) 2024-12-09T19:03:56,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/b2ee67c7a1004e379a8a7ef9db712814 2024-12-09T19:03:56,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/fbb7706248a9402f844a60e96f0305fc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc 2024-12-09T19:03:56,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc, entries=200, sequenceid=15, filesize=38.6 K 2024-12-09T19:03:56,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/c1781e3c31d741b59b3af791fe805c67 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/c1781e3c31d741b59b3af791fe805c67 2024-12-09T19:03:56,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/c1781e3c31d741b59b3af791fe805c67, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T19:03:56,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/b2ee67c7a1004e379a8a7ef9db712814 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/b2ee67c7a1004e379a8a7ef9db712814 2024-12-09T19:03:56,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/b2ee67c7a1004e379a8a7ef9db712814, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T19:03:56,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 524e93dcb7c45b6722e1741895f65463 in 255ms, sequenceid=15, compaction requested=false 2024-12-09T19:03:56,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:56,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-09T19:03:56,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:56,953 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:03:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T19:03:56,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab548ed2ebe04f979d25bf8f6a3cc15c_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771036699/Put/seqid=0 2024-12-09T19:03:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741943_1119 (size=12154) 2024-12-09T19:03:56,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:56,992 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ab548ed2ebe04f979d25bf8f6a3cc15c_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab548ed2ebe04f979d25bf8f6a3cc15c_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0d05f9ab4aa24822929f05534b63ad63, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0d05f9ab4aa24822929f05534b63ad63 is 175, key is test_row_0/A:col10/1733771036699/Put/seqid=0 2024-12-09T19:03:57,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741944_1120 (size=30955) 2024-12-09T19:03:57,019 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0d05f9ab4aa24822929f05534b63ad63 2024-12-09T19:03:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/533dd61f3afc43ccab6d4076606918d8 is 50, key is test_row_0/B:col10/1733771036699/Put/seqid=0 2024-12-09T19:03:57,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741945_1121 (size=12001) 2024-12-09T19:03:57,052 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/533dd61f3afc43ccab6d4076606918d8 2024-12-09T19:03:57,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771097042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771097050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771097053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771097056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771097057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/feae470929ef498684bcc0aa5570fe26 is 50, key is test_row_0/C:col10/1733771036699/Put/seqid=0 2024-12-09T19:03:57,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741946_1122 (size=12001) 2024-12-09T19:03:57,085 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/feae470929ef498684bcc0aa5570fe26 2024-12-09T19:03:57,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0d05f9ab4aa24822929f05534b63ad63 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63 2024-12-09T19:03:57,104 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63, entries=150, sequenceid=40, filesize=30.2 K 2024-12-09T19:03:57,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/533dd61f3afc43ccab6d4076606918d8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/533dd61f3afc43ccab6d4076606918d8 2024-12-09T19:03:57,113 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/533dd61f3afc43ccab6d4076606918d8, entries=150, sequenceid=40, filesize=11.7 K 2024-12-09T19:03:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/feae470929ef498684bcc0aa5570fe26 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/feae470929ef498684bcc0aa5570fe26 2024-12-09T19:03:57,126 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/feae470929ef498684bcc0aa5570fe26, entries=150, sequenceid=40, filesize=11.7 K 2024-12-09T19:03:57,128 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 524e93dcb7c45b6722e1741895f65463 in 176ms, sequenceid=40, compaction requested=false 2024-12-09T19:03:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-09T19:03:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-09T19:03:57,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-09T19:03:57,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 490 msec 2024-12-09T19:03:57,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 502 msec 2024-12-09T19:03:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:57,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-09T19:03:57,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120980e6ac915e2047ce91068d48779869be_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:57,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741947_1123 (size=17034) 2024-12-09T19:03:57,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771097240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771097245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771097245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771097246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771097248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-09T19:03:57,265 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:57,266 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T19:03:57,268 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T19:03:57,269 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:57,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:57,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771097355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771097365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771097365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771097365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771097365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T19:03:57,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771097565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771097574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771097574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771097574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771097575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:57,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:57,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T19:03:57,615 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:57,622 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120980e6ac915e2047ce91068d48779869be_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120980e6ac915e2047ce91068d48779869be_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:57,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e8264e10699f4a51bc120d79bd6040a0, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:57,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e8264e10699f4a51bc120d79bd6040a0 is 175, key is test_row_0/A:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:57,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741948_1124 (size=48139) 2024-12-09T19:03:57,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771097874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771097885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771097885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771097885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771097885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:57,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:57,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:57,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:57,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:57,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T19:03:58,039 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e8264e10699f4a51bc120d79bd6040a0 2024-12-09T19:03:58,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/fc3b9a1e28f049369f9892cf44a3e26a is 50, key is test_row_0/B:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:58,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741949_1125 (size=12001) 2024-12-09T19:03:58,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,258 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:03:58,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771098385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771098394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771098394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:58,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771098395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:58,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771098395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T19:03:58,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/fc3b9a1e28f049369f9892cf44a3e26a 2024-12-09T19:03:58,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/2526ca4fefd84be3836f18ef70a6693a is 50, key is test_row_0/C:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:58,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741950_1126 (size=12001) 2024-12-09T19:03:58,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/2526ca4fefd84be3836f18ef70a6693a 2024-12-09T19:03:58,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e8264e10699f4a51bc120d79bd6040a0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0 2024-12-09T19:03:58,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0, entries=250, sequenceid=54, filesize=47.0 K 2024-12-09T19:03:58,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/fc3b9a1e28f049369f9892cf44a3e26a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/fc3b9a1e28f049369f9892cf44a3e26a 2024-12-09T19:03:58,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:58,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/fc3b9a1e28f049369f9892cf44a3e26a, entries=150, sequenceid=54, filesize=11.7 K 2024-12-09T19:03:58,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/2526ca4fefd84be3836f18ef70a6693a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/2526ca4fefd84be3836f18ef70a6693a 2024-12-09T19:03:58,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/2526ca4fefd84be3836f18ef70a6693a, entries=150, sequenceid=54, filesize=11.7 K 2024-12-09T19:03:58,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 524e93dcb7c45b6722e1741895f65463 in 1346ms, sequenceid=54, compaction requested=true 2024-12-09T19:03:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:58,524 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:03:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:58,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:03:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:03:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:58,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:58,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/A is initiating minor compaction (all files) 2024-12-09T19:03:58,526 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/A in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,526 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=115.9 K 2024-12-09T19:03:58,526 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,527 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0] 2024-12-09T19:03:58,527 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:58,527 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/B is initiating minor compaction (all files) 2024-12-09T19:03:58,527 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/B in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,527 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/c1781e3c31d741b59b3af791fe805c67, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/533dd61f3afc43ccab6d4076606918d8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/fc3b9a1e28f049369f9892cf44a3e26a] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.2 K 2024-12-09T19:03:58,528 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting c1781e3c31d741b59b3af791fe805c67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733771036648 2024-12-09T19:03:58,528 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb7706248a9402f844a60e96f0305fc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733771036641 2024-12-09T19:03:58,529 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 533dd61f3afc43ccab6d4076606918d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733771036699 2024-12-09T19:03:58,529 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d05f9ab4aa24822929f05534b63ad63, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733771036699 2024-12-09T19:03:58,530 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3b9a1e28f049369f9892cf44a3e26a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733771037033 2024-12-09T19:03:58,530 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8264e10699f4a51bc120d79bd6040a0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733771037033 2024-12-09T19:03:58,545 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:58,546 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/313e8b6cf9b64ffd9a5664cbfbc67988 is 50, key is test_row_0/B:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:58,547 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:58,556 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e034f67d1c3143148f50b73ea6e78e82_524e93dcb7c45b6722e1741895f65463 store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:58,564 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e034f67d1c3143148f50b73ea6e78e82_524e93dcb7c45b6722e1741895f65463, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:58,565 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e034f67d1c3143148f50b73ea6e78e82_524e93dcb7c45b6722e1741895f65463 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741951_1127 (size=12104) 2024-12-09T19:03:58,598 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/313e8b6cf9b64ffd9a5664cbfbc67988 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/313e8b6cf9b64ffd9a5664cbfbc67988 2024-12-09T19:03:58,613 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/B of 524e93dcb7c45b6722e1741895f65463 into 313e8b6cf9b64ffd9a5664cbfbc67988(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:58,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:58,614 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/B, priority=13, startTime=1733771038524; duration=0sec 2024-12-09T19:03:58,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:03:58,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:03:58,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:03:58,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741952_1128 (size=4469) 2024-12-09T19:03:58,617 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#A#compaction#109 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:58,617 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/cfe27efa29544afba98d8ada080b453f is 175, key is test_row_0/A:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:58,618 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:03:58,619 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/C is initiating minor compaction (all files) 2024-12-09T19:03:58,619 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/C in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,619 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/b2ee67c7a1004e379a8a7ef9db712814, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/feae470929ef498684bcc0aa5570fe26, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/2526ca4fefd84be3836f18ef70a6693a] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.2 K 2024-12-09T19:03:58,622 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ee67c7a1004e379a8a7ef9db712814, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733771036648 2024-12-09T19:03:58,623 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting feae470929ef498684bcc0aa5570fe26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733771036699 2024-12-09T19:03:58,625 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2526ca4fefd84be3836f18ef70a6693a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733771037033 2024-12-09T19:03:58,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741953_1129 (size=31058) 2024-12-09T19:03:58,644 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/cfe27efa29544afba98d8ada080b453f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/cfe27efa29544afba98d8ada080b453f 2024-12-09T19:03:58,645 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#C#compaction#110 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:03:58,646 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/98f749dce5e34aa597019702b842ec01 is 50, key is test_row_0/C:col10/1733771037173/Put/seqid=0 2024-12-09T19:03:58,651 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/A of 524e93dcb7c45b6722e1741895f65463 into cfe27efa29544afba98d8ada080b453f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:58,651 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:58,652 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/A, priority=13, startTime=1733771038524; duration=0sec 2024-12-09T19:03:58,652 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:58,652 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:03:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741954_1130 (size=12104) 2024-12-09T19:03:58,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T19:03:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:58,661 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-09T19:03:58,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091896e9bf80c045509b7309a600f54170_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771037244/Put/seqid=0 2024-12-09T19:03:58,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741955_1131 (size=12154) 2024-12-09T19:03:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:58,697 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091896e9bf80c045509b7309a600f54170_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091896e9bf80c045509b7309a600f54170_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/33c77b14f5054d02856533f85f38d6ad, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/33c77b14f5054d02856533f85f38d6ad is 175, key is test_row_0/A:col10/1733771037244/Put/seqid=0 2024-12-09T19:03:58,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741956_1132 (size=30955) 2024-12-09T19:03:59,064 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/98f749dce5e34aa597019702b842ec01 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/98f749dce5e34aa597019702b842ec01 2024-12-09T19:03:59,073 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/C of 524e93dcb7c45b6722e1741895f65463 into 98f749dce5e34aa597019702b842ec01(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:03:59,073 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:59,073 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/C, priority=13, startTime=1733771038524; duration=0sec 2024-12-09T19:03:59,073 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:03:59,073 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:03:59,117 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/33c77b14f5054d02856533f85f38d6ad 2024-12-09T19:03:59,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5c4fd106e92d48308a8a95aef43e1499 is 50, key is test_row_0/B:col10/1733771037244/Put/seqid=0 2024-12-09T19:03:59,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741957_1133 (size=12001) 2024-12-09T19:03:59,140 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5c4fd106e92d48308a8a95aef43e1499 2024-12-09T19:03:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a7cca53341bb4fa4b137ca050c4e4d11 is 50, key is test_row_0/C:col10/1733771037244/Put/seqid=0 2024-12-09T19:03:59,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741958_1134 (size=12001) 2024-12-09T19:03:59,175 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a7cca53341bb4fa4b137ca050c4e4d11 2024-12-09T19:03:59,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/33c77b14f5054d02856533f85f38d6ad as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad 2024-12-09T19:03:59,195 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad, entries=150, sequenceid=78, filesize=30.2 K 2024-12-09T19:03:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5c4fd106e92d48308a8a95aef43e1499 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c4fd106e92d48308a8a95aef43e1499 2024-12-09T19:03:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,204 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c4fd106e92d48308a8a95aef43e1499, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:03:59,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a7cca53341bb4fa4b137ca050c4e4d11 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a7cca53341bb4fa4b137ca050c4e4d11 2024-12-09T19:03:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,229 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a7cca53341bb4fa4b137ca050c4e4d11, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:03:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,230 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 524e93dcb7c45b6722e1741895f65463 in 569ms, sequenceid=78, compaction requested=false 2024-12-09T19:03:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:03:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T19:03:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T19:03:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T19:03:59,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-12-09T19:03:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9700 sec 2024-12-09T19:03:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T19:03:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,415 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:03:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,416 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:03:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T19:03:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:03:59,418 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:03:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:03:59,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:03:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:59,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:03:59,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:03:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094fcc2d6791fa42cfba7a195899a4ccab_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771039455/Put/seqid=0 2024-12-09T19:03:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741960_1136 (size=24358) 2024-12-09T19:03:59,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771099512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771099512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771099512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771099514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771099516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:03:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:03:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771099625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771099625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771099627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771099627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771099634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:03:59,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:03:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771099834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771099834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771099835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771099835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:03:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771099844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:03:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:03:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:03:59,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:03:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:03:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:03:59,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:03:59,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:03:59,923 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094fcc2d6791fa42cfba7a195899a4ccab_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094fcc2d6791fa42cfba7a195899a4ccab_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:03:59,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/f4fd6764c99a46bf9d993020c7d55bbb, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:03:59,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/f4fd6764c99a46bf9d993020c7d55bbb is 175, key is test_row_0/A:col10/1733771039455/Put/seqid=0 2024-12-09T19:03:59,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741959_1135 (size=73995) 2024-12-09T19:04:00,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:04:00,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:00,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:04:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771100145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771100145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771100145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771100145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771100154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:04:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/f4fd6764c99a46bf9d993020c7d55bbb 2024-12-09T19:04:00,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/93e3f3a643e84a218ddad9f767dd7453 is 50, key is test_row_0/B:col10/1733771039455/Put/seqid=0 2024-12-09T19:04:00,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:04:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741961_1137 (size=12001) 2024-12-09T19:04:00,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/93e3f3a643e84a218ddad9f767dd7453 2024-12-09T19:04:00,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/81c2c3ec40a0495d8eed4787a1574816 is 50, key is test_row_0/C:col10/1733771039455/Put/seqid=0 2024-12-09T19:04:00,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741962_1138 (size=12001) 2024-12-09T19:04:00,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/81c2c3ec40a0495d8eed4787a1574816 2024-12-09T19:04:00,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/f4fd6764c99a46bf9d993020c7d55bbb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb 2024-12-09T19:04:00,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb, entries=400, sequenceid=90, filesize=72.3 K 2024-12-09T19:04:00,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/93e3f3a643e84a218ddad9f767dd7453 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/93e3f3a643e84a218ddad9f767dd7453 2024-12-09T19:04:00,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/93e3f3a643e84a218ddad9f767dd7453, entries=150, sequenceid=90, filesize=11.7 K 2024-12-09T19:04:00,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/81c2c3ec40a0495d8eed4787a1574816 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/81c2c3ec40a0495d8eed4787a1574816 2024-12-09T19:04:00,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/81c2c3ec40a0495d8eed4787a1574816, entries=150, sequenceid=90, filesize=11.7 K 2024-12-09T19:04:00,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 524e93dcb7c45b6722e1741895f65463 in 931ms, sequenceid=90, compaction requested=true 2024-12-09T19:04:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:00,389 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:00,389 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:00,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:00,393 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136008 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:00,393 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/A is initiating minor compaction (all files) 2024-12-09T19:04:00,393 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/A in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,393 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/cfe27efa29544afba98d8ada080b453f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=132.8 K 2024-12-09T19:04:00,393 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,393 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/cfe27efa29544afba98d8ada080b453f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb] 2024-12-09T19:04:00,394 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe27efa29544afba98d8ada080b453f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733771037033 2024-12-09T19:04:00,394 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:00,394 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/B is initiating minor compaction (all files) 2024-12-09T19:04:00,395 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/B in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,395 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/313e8b6cf9b64ffd9a5664cbfbc67988, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c4fd106e92d48308a8a95aef43e1499, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/93e3f3a643e84a218ddad9f767dd7453] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.3 K 2024-12-09T19:04:00,395 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c77b14f5054d02856533f85f38d6ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771037238 2024-12-09T19:04:00,395 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 313e8b6cf9b64ffd9a5664cbfbc67988, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733771037033 2024-12-09T19:04:00,396 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c4fd106e92d48308a8a95aef43e1499, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771037238 2024-12-09T19:04:00,396 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e3f3a643e84a218ddad9f767dd7453, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733771039447 2024-12-09T19:04:00,396 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fd6764c99a46bf9d993020c7d55bbb, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733771039424 2024-12-09T19:04:00,408 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:00,408 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:00,408 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/7132fa23a4fa4b19abc8091cb2686a1d is 50, key is test_row_0/B:col10/1733771039455/Put/seqid=0 2024-12-09T19:04:00,411 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412099b971acc8580443ea9ca7e1aa3066875_524e93dcb7c45b6722e1741895f65463 store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:00,413 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412099b971acc8580443ea9ca7e1aa3066875_524e93dcb7c45b6722e1741895f65463, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:00,413 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099b971acc8580443ea9ca7e1aa3066875_524e93dcb7c45b6722e1741895f65463 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:00,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741963_1139 (size=12207) 2024-12-09T19:04:00,436 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/7132fa23a4fa4b19abc8091cb2686a1d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/7132fa23a4fa4b19abc8091cb2686a1d 2024-12-09T19:04:00,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741964_1140 (size=4469) 2024-12-09T19:04:00,445 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#A#compaction#118 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:00,445 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/B of 524e93dcb7c45b6722e1741895f65463 into 7132fa23a4fa4b19abc8091cb2686a1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:00,445 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:00,446 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/B, priority=13, startTime=1733771040389; duration=0sec 2024-12-09T19:04:00,446 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:00,446 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:04:00,446 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/4c407c659f324f6586211469e4589e8f is 175, key is test_row_0/A:col10/1733771039455/Put/seqid=0 2024-12-09T19:04:00,446 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:00,448 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:00,449 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/C is initiating minor compaction (all files) 2024-12-09T19:04:00,449 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/C in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,449 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/98f749dce5e34aa597019702b842ec01, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a7cca53341bb4fa4b137ca050c4e4d11, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/81c2c3ec40a0495d8eed4787a1574816] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.3 K 2024-12-09T19:04:00,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741965_1141 (size=31161) 2024-12-09T19:04:00,450 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f749dce5e34aa597019702b842ec01, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733771037033 2024-12-09T19:04:00,450 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a7cca53341bb4fa4b137ca050c4e4d11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771037238 2024-12-09T19:04:00,451 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c2c3ec40a0495d8eed4787a1574816, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733771039447 2024-12-09T19:04:00,459 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/4c407c659f324f6586211469e4589e8f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/4c407c659f324f6586211469e4589e8f 2024-12-09T19:04:00,468 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/A of 524e93dcb7c45b6722e1741895f65463 into 4c407c659f324f6586211469e4589e8f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:00,468 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:00,468 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/A, priority=13, startTime=1733771040389; duration=0sec 2024-12-09T19:04:00,468 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:00,468 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:04:00,480 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:00,481 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/38d49387c1bb4aa2a71c4b107bbe8bb2 is 50, key is test_row_0/C:col10/1733771039455/Put/seqid=0 2024-12-09T19:04:00,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T19:04:00,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:00,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741966_1142 (size=12207) 2024-12-09T19:04:00,493 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:04:00,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/38d49387c1bb4aa2a71c4b107bbe8bb2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/38d49387c1bb4aa2a71c4b107bbe8bb2 2024-12-09T19:04:00,514 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/C of 524e93dcb7c45b6722e1741895f65463 into 38d49387c1bb4aa2a71c4b107bbe8bb2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:00,514 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:00,515 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/C, priority=13, startTime=1733771040390; duration=0sec 2024-12-09T19:04:00,515 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:00,515 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:04:00,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099c5a30ead234438d8c64e9334ab7718b_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771039513/Put/seqid=0 2024-12-09T19:04:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741967_1143 (size=12154) 2024-12-09T19:04:00,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:00,542 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099c5a30ead234438d8c64e9334ab7718b_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099c5a30ead234438d8c64e9334ab7718b_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:00,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/36e41ffeee67493d8308f237d6ea15e8, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:00,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/36e41ffeee67493d8308f237d6ea15e8 is 175, key is test_row_0/A:col10/1733771039513/Put/seqid=0 2024-12-09T19:04:00,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741968_1144 (size=30955) 2024-12-09T19:04:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:04:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:00,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771100661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771100662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771100663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771100664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771100674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771100775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771100775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771100775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771100775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,949 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/36e41ffeee67493d8308f237d6ea15e8 2024-12-09T19:04:00,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/28aa8848b5ca4bb59690e12e5b8066b0 is 50, key is test_row_0/B:col10/1733771039513/Put/seqid=0 2024-12-09T19:04:00,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741969_1145 (size=12001) 2024-12-09T19:04:00,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771100985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771100985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771100985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771100985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771101294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771101294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771101294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771101295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,377 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/28aa8848b5ca4bb59690e12e5b8066b0 2024-12-09T19:04:01,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/bba82437df1443668f9e4eb8e8deabd4 is 50, key is test_row_0/C:col10/1733771039513/Put/seqid=0 2024-12-09T19:04:01,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741970_1146 (size=12001) 2024-12-09T19:04:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:04:01,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771101685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,686 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771101804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771101804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771101804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771101804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,809 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/bba82437df1443668f9e4eb8e8deabd4 2024-12-09T19:04:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:01,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/36e41ffeee67493d8308f237d6ea15e8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8 2024-12-09T19:04:01,821 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8, entries=150, sequenceid=117, filesize=30.2 K 2024-12-09T19:04:01,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/28aa8848b5ca4bb59690e12e5b8066b0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/28aa8848b5ca4bb59690e12e5b8066b0 2024-12-09T19:04:01,829 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/28aa8848b5ca4bb59690e12e5b8066b0, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T19:04:01,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/bba82437df1443668f9e4eb8e8deabd4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/bba82437df1443668f9e4eb8e8deabd4 2024-12-09T19:04:01,837 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/bba82437df1443668f9e4eb8e8deabd4, entries=150, sequenceid=117, filesize=11.7 K 2024-12-09T19:04:01,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 524e93dcb7c45b6722e1741895f65463 in 1345ms, sequenceid=117, compaction requested=false 2024-12-09T19:04:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T19:04:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T19:04:01,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T19:04:01,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-12-09T19:04:01,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4270 sec 2024-12-09T19:04:02,706 INFO [master/a437f8b9ba7d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T19:04:02,706 INFO [master/a437f8b9ba7d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T19:04:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:02,816 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:02,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:04:02,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8902a253954450ab1ddd4d514ef50b6_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:02,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741971_1147 (size=12204) 2024-12-09T19:04:02,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771102850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771102851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771102852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771102853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771102965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771102965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771102965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771102965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771103175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771103175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771103175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771103175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,229 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:03,232 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8902a253954450ab1ddd4d514ef50b6_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8902a253954450ab1ddd4d514ef50b6_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:03,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/56654be8d42147d6ab4791f18511deb7, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:03,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/56654be8d42147d6ab4791f18511deb7 is 175, key is test_row_0/A:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:03,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741972_1148 (size=31005) 2024-12-09T19:04:03,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771103484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771103485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771103494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771103494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T19:04:03,585 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:03,586 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T19:04:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T19:04:03,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:03,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:03,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:03,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/56654be8d42147d6ab4791f18511deb7 2024-12-09T19:04:03,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/1605813e6b254b67b50a1c8afa9320ef is 50, key is test_row_0/B:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741973_1149 (size=12051) 2024-12-09T19:04:03,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/1605813e6b254b67b50a1c8afa9320ef 2024-12-09T19:04:03,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1e914cd9e78c4c82bc10b76ce94e3f3c is 50, key is test_row_0/C:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:03,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741974_1150 (size=12051) 2024-12-09T19:04:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T19:04:03,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771103694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,699 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:03,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T19:04:03,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:03,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:03,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T19:04:03,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:03,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:03,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T19:04:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771103995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771103995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:04,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771104004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:04,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771104004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:04,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T19:04:04,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:04,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:04,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:04,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1e914cd9e78c4c82bc10b76ce94e3f3c 2024-12-09T19:04:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/56654be8d42147d6ab4791f18511deb7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7 2024-12-09T19:04:04,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7, entries=150, sequenceid=130, filesize=30.3 K 2024-12-09T19:04:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/1605813e6b254b67b50a1c8afa9320ef as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/1605813e6b254b67b50a1c8afa9320ef 2024-12-09T19:04:04,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/1605813e6b254b67b50a1c8afa9320ef, entries=150, sequenceid=130, filesize=11.8 K 2024-12-09T19:04:04,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1e914cd9e78c4c82bc10b76ce94e3f3c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1e914cd9e78c4c82bc10b76ce94e3f3c 2024-12-09T19:04:04,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1e914cd9e78c4c82bc10b76ce94e3f3c, entries=150, sequenceid=130, filesize=11.8 K 2024-12-09T19:04:04,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 524e93dcb7c45b6722e1741895f65463 in 1284ms, sequenceid=130, compaction requested=true 2024-12-09T19:04:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:04,100 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:04,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:04,100 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:04,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:04,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:04,102 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93121 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:04,102 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/A is initiating minor compaction (all files) 2024-12-09T19:04:04,103 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/A in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,103 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/4c407c659f324f6586211469e4589e8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=90.9 K 2024-12-09T19:04:04,103 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,103 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/4c407c659f324f6586211469e4589e8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7] 2024-12-09T19:04:04,103 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:04,103 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/B is initiating minor compaction (all files) 2024-12-09T19:04:04,103 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/B in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,103 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c407c659f324f6586211469e4589e8f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733771039447 2024-12-09T19:04:04,103 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/7132fa23a4fa4b19abc8091cb2686a1d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/28aa8848b5ca4bb59690e12e5b8066b0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/1605813e6b254b67b50a1c8afa9320ef] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.4 K 2024-12-09T19:04:04,104 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e41ffeee67493d8308f237d6ea15e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733771039506 2024-12-09T19:04:04,104 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 7132fa23a4fa4b19abc8091cb2686a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733771039447 2024-12-09T19:04:04,104 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56654be8d42147d6ab4791f18511deb7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771040661 2024-12-09T19:04:04,104 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 28aa8848b5ca4bb59690e12e5b8066b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733771039506 2024-12-09T19:04:04,105 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1605813e6b254b67b50a1c8afa9320ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771040661 2024-12-09T19:04:04,112 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:04,113 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#B#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:04,113 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/cd7c6cafe62c4d22a07c2194f29ac488 is 50, key is test_row_0/B:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:04,115 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090c528229ae78486c96ed450cd6a138a9_524e93dcb7c45b6722e1741895f65463 store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:04,117 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090c528229ae78486c96ed450cd6a138a9_524e93dcb7c45b6722e1741895f65463, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:04,117 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090c528229ae78486c96ed450cd6a138a9_524e93dcb7c45b6722e1741895f65463 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:04,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741976_1152 (size=4469) 2024-12-09T19:04:04,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741975_1151 (size=12359) 2024-12-09T19:04:04,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T19:04:04,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,200 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:04:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120967287163119147929f5b728df9317bec_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771042849/Put/seqid=0 2024-12-09T19:04:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T19:04:04,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741977_1153 (size=12304) 2024-12-09T19:04:04,534 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#A#compaction#127 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:04,534 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/529aa535492e4e94b91c7f129990bfca is 175, key is test_row_0/A:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:04,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741978_1154 (size=31313) 2024-12-09T19:04:04,561 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/cd7c6cafe62c4d22a07c2194f29ac488 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/cd7c6cafe62c4d22a07c2194f29ac488 2024-12-09T19:04:04,566 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/B of 524e93dcb7c45b6722e1741895f65463 into cd7c6cafe62c4d22a07c2194f29ac488(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:04,566 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:04,566 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/B, priority=13, startTime=1733771044100; duration=0sec 2024-12-09T19:04:04,566 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:04,566 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:04:04,566 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:04,567 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:04,567 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/C is initiating minor compaction (all files) 2024-12-09T19:04:04,568 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/C in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:04,568 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/38d49387c1bb4aa2a71c4b107bbe8bb2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/bba82437df1443668f9e4eb8e8deabd4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1e914cd9e78c4c82bc10b76ce94e3f3c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.4 K 2024-12-09T19:04:04,568 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d49387c1bb4aa2a71c4b107bbe8bb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733771039447 2024-12-09T19:04:04,568 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting bba82437df1443668f9e4eb8e8deabd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733771039506 2024-12-09T19:04:04,569 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e914cd9e78c4c82bc10b76ce94e3f3c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771040661 2024-12-09T19:04:04,575 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:04,576 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a10b63b0df8949fbb717d0467a09df2b is 50, key is test_row_0/C:col10/1733771040661/Put/seqid=0 2024-12-09T19:04:04,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741979_1155 (size=12359) 2024-12-09T19:04:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:04,624 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120967287163119147929f5b728df9317bec_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120967287163119147929f5b728df9317bec_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/30a929afec5646fcbbceeb76efba97f9, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/30a929afec5646fcbbceeb76efba97f9 is 175, key is test_row_0/A:col10/1733771042849/Put/seqid=0 2024-12-09T19:04:04,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741980_1156 (size=31105) 2024-12-09T19:04:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T19:04:04,964 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/529aa535492e4e94b91c7f129990bfca as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/529aa535492e4e94b91c7f129990bfca 2024-12-09T19:04:04,969 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/A of 524e93dcb7c45b6722e1741895f65463 into 529aa535492e4e94b91c7f129990bfca(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:04,969 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:04,969 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/A, priority=13, startTime=1733771044100; duration=0sec 2024-12-09T19:04:04,969 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:04,970 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:04:04,995 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a10b63b0df8949fbb717d0467a09df2b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a10b63b0df8949fbb717d0467a09df2b 2024-12-09T19:04:05,000 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/C of 524e93dcb7c45b6722e1741895f65463 into a10b63b0df8949fbb717d0467a09df2b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:05,000 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:05,000 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/C, priority=13, startTime=1733771044100; duration=0sec 2024-12-09T19:04:05,000 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:05,000 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:04:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:05,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:05,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771105025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771105029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,030 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/30a929afec5646fcbbceeb76efba97f9 2024-12-09T19:04:05,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771105029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771105030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/50380a3243ae41bfb754214c4448f4e2 is 50, key is test_row_0/B:col10/1733771042849/Put/seqid=0 2024-12-09T19:04:05,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741981_1157 (size=12151) 2024-12-09T19:04:05,053 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/50380a3243ae41bfb754214c4448f4e2 2024-12-09T19:04:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/45b8fbd87b104aca803b704a8ac95c22 is 50, key is test_row_0/C:col10/1733771042849/Put/seqid=0 2024-12-09T19:04:05,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741982_1158 (size=12151) 2024-12-09T19:04:05,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771105135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771105135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771105135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771105144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771105345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771105345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771105345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771105354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,467 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/45b8fbd87b104aca803b704a8ac95c22 2024-12-09T19:04:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/30a929afec5646fcbbceeb76efba97f9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9 2024-12-09T19:04:05,476 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9, entries=150, sequenceid=154, filesize=30.4 K 2024-12-09T19:04:05,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/50380a3243ae41bfb754214c4448f4e2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/50380a3243ae41bfb754214c4448f4e2 2024-12-09T19:04:05,481 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/50380a3243ae41bfb754214c4448f4e2, entries=150, sequenceid=154, filesize=11.9 K 2024-12-09T19:04:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/45b8fbd87b104aca803b704a8ac95c22 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/45b8fbd87b104aca803b704a8ac95c22 2024-12-09T19:04:05,487 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/45b8fbd87b104aca803b704a8ac95c22, entries=150, sequenceid=154, filesize=11.9 K 2024-12-09T19:04:05,488 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 524e93dcb7c45b6722e1741895f65463 in 1288ms, sequenceid=154, compaction requested=false 2024-12-09T19:04:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:05,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T19:04:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T19:04:05,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T19:04:05,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9010 sec 2024-12-09T19:04:05,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9060 sec 2024-12-09T19:04:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:05,656 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:05,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-09T19:04:05,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097ae2b9eb66a44a83aff58c87ec7876f3_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:05,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741983_1159 (size=19774) 2024-12-09T19:04:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771105703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771105703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771105704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771105704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T19:04:05,734 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:05,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T19:04:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:05,737 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:05,738 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:05,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:05,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771105815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771105815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771105815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771105815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:05,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:05,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:05,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771106024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771106024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771106024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771106024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:06,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:06,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097ae2b9eb66a44a83aff58c87ec7876f3_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097ae2b9eb66a44a83aff58c87ec7876f3_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:06,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/6b54e46a1aea498d9bac27de5548c593, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:06,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/6b54e46a1aea498d9bac27de5548c593 is 175, key is test_row_0/A:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741984_1160 (size=57033) 2024-12-09T19:04:06,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771106334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771106334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771106334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771106334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:06,485 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/6b54e46a1aea498d9bac27de5548c593 2024-12-09T19:04:06,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/4944e9c3088b4eb58409607e3bfe1dba is 50, key is test_row_0/B:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:06,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741985_1161 (size=12151) 2024-12-09T19:04:06,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771106844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771106844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771106854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771106854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:06,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/4944e9c3088b4eb58409607e3bfe1dba 2024-12-09T19:04:06,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/761c75c2a81b45df8198f8318fe18156 is 50, key is test_row_0/C:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:06,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741986_1162 (size=12151) 2024-12-09T19:04:06,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:06,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:06,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:07,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:07,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:07,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/761c75c2a81b45df8198f8318fe18156 2024-12-09T19:04:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/6b54e46a1aea498d9bac27de5548c593 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593 2024-12-09T19:04:07,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593, entries=300, sequenceid=173, filesize=55.7 K 2024-12-09T19:04:07,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/4944e9c3088b4eb58409607e3bfe1dba as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/4944e9c3088b4eb58409607e3bfe1dba 2024-12-09T19:04:07,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/4944e9c3088b4eb58409607e3bfe1dba, entries=150, sequenceid=173, filesize=11.9 K 2024-12-09T19:04:07,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/761c75c2a81b45df8198f8318fe18156 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/761c75c2a81b45df8198f8318fe18156 2024-12-09T19:04:07,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/761c75c2a81b45df8198f8318fe18156, entries=150, sequenceid=173, filesize=11.9 K 2024-12-09T19:04:07,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 524e93dcb7c45b6722e1741895f65463 in 1678ms, sequenceid=173, compaction requested=true 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:07,334 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:07,334 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:07,335 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:07,335 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:07,335 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/A is initiating minor compaction (all files) 2024-12-09T19:04:07,335 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/B is initiating minor compaction (all files) 2024-12-09T19:04:07,335 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/A in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,335 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/B in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,336 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/529aa535492e4e94b91c7f129990bfca, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=116.7 K 2024-12-09T19:04:07,336 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/cd7c6cafe62c4d22a07c2194f29ac488, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/50380a3243ae41bfb754214c4448f4e2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/4944e9c3088b4eb58409607e3bfe1dba] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.8 K 2024-12-09T19:04:07,336 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,336 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/529aa535492e4e94b91c7f129990bfca, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593] 2024-12-09T19:04:07,336 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7c6cafe62c4d22a07c2194f29ac488, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771040661 2024-12-09T19:04:07,336 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529aa535492e4e94b91c7f129990bfca, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771040661 2024-12-09T19:04:07,336 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 50380a3243ae41bfb754214c4448f4e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771042849 2024-12-09T19:04:07,336 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30a929afec5646fcbbceeb76efba97f9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771042849 2024-12-09T19:04:07,337 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 4944e9c3088b4eb58409607e3bfe1dba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733771045655 2024-12-09T19:04:07,337 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b54e46a1aea498d9bac27de5548c593, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733771045022 2024-12-09T19:04:07,344 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:07,345 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5938079b3f874c409e9d0799d67867d9 is 50, key is test_row_0/B:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:07,346 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:07,348 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209d53159e5932c41bd84be8077e3bf63bd_524e93dcb7c45b6722e1741895f65463 store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:07,350 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209d53159e5932c41bd84be8077e3bf63bd_524e93dcb7c45b6722e1741895f65463, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:07,350 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d53159e5932c41bd84be8077e3bf63bd_524e93dcb7c45b6722e1741895f65463 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741987_1163 (size=12561) 2024-12-09T19:04:07,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741988_1164 (size=4469) 2024-12-09T19:04:07,364 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#A#compaction#136 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:07,365 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/5e8de4d6c2ab472380841b702ab3a74c is 175, key is test_row_0/A:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:07,365 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5938079b3f874c409e9d0799d67867d9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5938079b3f874c409e9d0799d67867d9 2024-12-09T19:04:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741989_1165 (size=31515) 2024-12-09T19:04:07,375 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/B of 524e93dcb7c45b6722e1741895f65463 into 5938079b3f874c409e9d0799d67867d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:07,375 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:07,375 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/B, priority=13, startTime=1733771047334; duration=0sec 2024-12-09T19:04:07,376 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:07,376 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:04:07,376 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:07,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:07,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/C is initiating minor compaction (all files) 2024-12-09T19:04:07,379 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/C in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,379 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a10b63b0df8949fbb717d0467a09df2b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/45b8fbd87b104aca803b704a8ac95c22, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/761c75c2a81b45df8198f8318fe18156] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=35.8 K 2024-12-09T19:04:07,379 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a10b63b0df8949fbb717d0467a09df2b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771040661 2024-12-09T19:04:07,379 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/5e8de4d6c2ab472380841b702ab3a74c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/5e8de4d6c2ab472380841b702ab3a74c 2024-12-09T19:04:07,380 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b8fbd87b104aca803b704a8ac95c22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771042849 2024-12-09T19:04:07,380 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 761c75c2a81b45df8198f8318fe18156, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733771045655 2024-12-09T19:04:07,386 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/A of 524e93dcb7c45b6722e1741895f65463 into 5e8de4d6c2ab472380841b702ab3a74c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:07,386 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:07,386 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/A, priority=13, startTime=1733771047334; duration=0sec 2024-12-09T19:04:07,387 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:07,387 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:04:07,390 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#C#compaction#137 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:07,391 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1ffa1932e74b4e2ebc782eca5fef666e is 50, key is test_row_0/C:col10/1733771045656/Put/seqid=0 2024-12-09T19:04:07,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741990_1166 (size=12561) 2024-12-09T19:04:07,407 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1ffa1932e74b4e2ebc782eca5fef666e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1ffa1932e74b4e2ebc782eca5fef666e 2024-12-09T19:04:07,415 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/C of 524e93dcb7c45b6722e1741895f65463 into 1ffa1932e74b4e2ebc782eca5fef666e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:07,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:07,415 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/C, priority=13, startTime=1733771047334; duration=0sec 2024-12-09T19:04:07,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:07,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:04:07,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T19:04:07,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,419 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-09T19:04:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f5848bb0cf1c458f8b14cb80e62de1d4_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771045702/Put/seqid=0 2024-12-09T19:04:07,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741991_1167 (size=12304) 2024-12-09T19:04:07,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:07,450 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f5848bb0cf1c458f8b14cb80e62de1d4_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f5848bb0cf1c458f8b14cb80e62de1d4_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e86256e1ac874f45a72c2a75caa5e1e9, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e86256e1ac874f45a72c2a75caa5e1e9 is 175, key is test_row_0/A:col10/1733771045702/Put/seqid=0 2024-12-09T19:04:07,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741992_1168 (size=31105) 2024-12-09T19:04:07,467 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e86256e1ac874f45a72c2a75caa5e1e9 2024-12-09T19:04:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5c747758a9b343f4b8232839a8f1023e is 50, key is test_row_0/B:col10/1733771045702/Put/seqid=0 2024-12-09T19:04:07,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741993_1169 (size=12151) 2024-12-09T19:04:07,479 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5c747758a9b343f4b8232839a8f1023e 2024-12-09T19:04:07,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1c094820b48a44e183de3683afc40e98 is 50, key is test_row_0/C:col10/1733771045702/Put/seqid=0 2024-12-09T19:04:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741994_1170 (size=12151) 2024-12-09T19:04:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:07,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771107782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771107854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:07,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771107854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:07,856 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:07,856 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:07,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771107864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:07,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771107864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:07,865 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:07,865 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771107894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:07,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:07,896 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1c094820b48a44e183de3683afc40e98 2024-12-09T19:04:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e86256e1ac874f45a72c2a75caa5e1e9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9 2024-12-09T19:04:07,905 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9, entries=150, sequenceid=196, filesize=30.4 K 2024-12-09T19:04:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/5c747758a9b343f4b8232839a8f1023e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c747758a9b343f4b8232839a8f1023e 2024-12-09T19:04:07,910 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c747758a9b343f4b8232839a8f1023e, entries=150, sequenceid=196, filesize=11.9 K 2024-12-09T19:04:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/1c094820b48a44e183de3683afc40e98 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1c094820b48a44e183de3683afc40e98 2024-12-09T19:04:07,917 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1c094820b48a44e183de3683afc40e98, entries=150, sequenceid=196, filesize=11.9 K 2024-12-09T19:04:07,918 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 524e93dcb7c45b6722e1741895f65463 in 500ms, sequenceid=196, compaction requested=false 2024-12-09T19:04:07,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:07,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:07,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T19:04:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T19:04:07,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T19:04:07,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-12-09T19:04:07,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.1880 sec 2024-12-09T19:04:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:08,106 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:08,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-09T19:04:08,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095564a290848f460e9a5bdd1c021caa2e_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:08,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741995_1171 (size=14794) 2024-12-09T19:04:08,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771108182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:08,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771108294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:08,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771108505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:08,525 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:08,528 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095564a290848f460e9a5bdd1c021caa2e_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095564a290848f460e9a5bdd1c021caa2e_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:08,529 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c88abf11ad17423da9af82232046922d, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:08,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c88abf11ad17423da9af82232046922d is 175, key is test_row_0/A:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:08,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741996_1172 (size=39749) 2024-12-09T19:04:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771108814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:08,934 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c88abf11ad17423da9af82232046922d 2024-12-09T19:04:08,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/46cfa4e6c8c744fbad66a6c000ca3a8f is 50, key is test_row_0/B:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:08,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741997_1173 (size=12151) 2024-12-09T19:04:09,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771109324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:09,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/46cfa4e6c8c744fbad66a6c000ca3a8f 2024-12-09T19:04:09,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/96c45f957cb2495e875c4cb8a6baf726 is 50, key is test_row_0/C:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:09,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741998_1174 (size=12151) 2024-12-09T19:04:09,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/96c45f957cb2495e875c4cb8a6baf726 2024-12-09T19:04:09,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c88abf11ad17423da9af82232046922d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d 2024-12-09T19:04:09,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d, entries=200, sequenceid=213, filesize=38.8 K 2024-12-09T19:04:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/46cfa4e6c8c744fbad66a6c000ca3a8f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/46cfa4e6c8c744fbad66a6c000ca3a8f 2024-12-09T19:04:09,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/46cfa4e6c8c744fbad66a6c000ca3a8f, entries=150, sequenceid=213, filesize=11.9 K 2024-12-09T19:04:09,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/96c45f957cb2495e875c4cb8a6baf726 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/96c45f957cb2495e875c4cb8a6baf726 2024-12-09T19:04:09,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/96c45f957cb2495e875c4cb8a6baf726, entries=150, sequenceid=213, filesize=11.9 K 2024-12-09T19:04:09,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 524e93dcb7c45b6722e1741895f65463 in 1696ms, sequenceid=213, compaction requested=true 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:09,802 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:09,802 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:09,803 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:09,803 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/B is initiating minor compaction (all files) 2024-12-09T19:04:09,804 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/B in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:09,804 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:09,804 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/A is initiating minor compaction (all files) 2024-12-09T19:04:09,804 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5938079b3f874c409e9d0799d67867d9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c747758a9b343f4b8232839a8f1023e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/46cfa4e6c8c744fbad66a6c000ca3a8f] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=36.0 K 2024-12-09T19:04:09,804 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/A in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:09,804 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/5e8de4d6c2ab472380841b702ab3a74c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=100.0 K 2024-12-09T19:04:09,804 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:09,804 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/5e8de4d6c2ab472380841b702ab3a74c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d] 2024-12-09T19:04:09,804 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5938079b3f874c409e9d0799d67867d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733771045655 2024-12-09T19:04:09,804 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8de4d6c2ab472380841b702ab3a74c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733771045655 2024-12-09T19:04:09,805 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86256e1ac874f45a72c2a75caa5e1e9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733771045701 2024-12-09T19:04:09,805 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c747758a9b343f4b8232839a8f1023e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733771045701 2024-12-09T19:04:09,805 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c88abf11ad17423da9af82232046922d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733771047759 2024-12-09T19:04:09,805 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 46cfa4e6c8c744fbad66a6c000ca3a8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733771047759 2024-12-09T19:04:09,816 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:09,818 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#B#compaction#145 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:09,818 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/31d8b84cd1f44309957b4a8d8fe22679 is 50, key is test_row_0/B:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:09,820 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209f94cc82c0d8948249568cf388d576424_524e93dcb7c45b6722e1741895f65463 store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:09,822 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209f94cc82c0d8948249568cf388d576424_524e93dcb7c45b6722e1741895f65463, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:09,822 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f94cc82c0d8948249568cf388d576424_524e93dcb7c45b6722e1741895f65463 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:09,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741999_1175 (size=12663) 2024-12-09T19:04:09,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742000_1176 (size=4469) 2024-12-09T19:04:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:09,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:09,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-09T19:04:09,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209160a9ea8fc8e464ea9499ed45985652d_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771048171/Put/seqid=0 2024-12-09T19:04:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742001_1177 (size=14794) 2024-12-09T19:04:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771109897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771109898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T19:04:09,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:09,905 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771109902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771109902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T19:04:09,907 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:09,908 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:09,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771110004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771110004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:10,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771110014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771110015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771110214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771110214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:10,228 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/31d8b84cd1f44309957b4a8d8fe22679 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/31d8b84cd1f44309957b4a8d8fe22679 2024-12-09T19:04:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771110225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,230 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:10,230 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/44103e89da4842b2ba32d1e620fd8e89 is 175, key is test_row_0/A:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771110225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,234 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/B of 524e93dcb7c45b6722e1741895f65463 into 31d8b84cd1f44309957b4a8d8fe22679(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:10,235 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:10,235 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/B, priority=13, startTime=1733771049802; duration=0sec 2024-12-09T19:04:10,235 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:10,235 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:04:10,235 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:10,236 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:10,236 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/C is initiating minor compaction (all files) 2024-12-09T19:04:10,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742002_1178 (size=31617) 2024-12-09T19:04:10,236 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/C in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,236 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1ffa1932e74b4e2ebc782eca5fef666e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1c094820b48a44e183de3683afc40e98, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/96c45f957cb2495e875c4cb8a6baf726] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=36.0 K 2024-12-09T19:04:10,237 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ffa1932e74b4e2ebc782eca5fef666e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1733771045655 2024-12-09T19:04:10,237 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c094820b48a44e183de3683afc40e98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733771045701 2024-12-09T19:04:10,238 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c45f957cb2495e875c4cb8a6baf726, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733771047759 2024-12-09T19:04:10,245 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:10,246 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/46b145be82d947e8aa817fc8058c28eb is 50, key is test_row_0/C:col10/1733771047780/Put/seqid=0 2024-12-09T19:04:10,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742003_1179 (size=12663) 2024-12-09T19:04:10,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:10,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209160a9ea8fc8e464ea9499ed45985652d_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209160a9ea8fc8e464ea9499ed45985652d_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:10,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/9b60a160392745af9b686490aa812498, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:10,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/9b60a160392745af9b686490aa812498 is 175, key is test_row_0/A:col10/1733771048171/Put/seqid=0 2024-12-09T19:04:10,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742004_1180 (size=39749) 2024-12-09T19:04:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771110334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,337 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771110525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771110525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:10,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771110535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771110544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:10,642 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/44103e89da4842b2ba32d1e620fd8e89 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/44103e89da4842b2ba32d1e620fd8e89 2024-12-09T19:04:10,647 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/A of 524e93dcb7c45b6722e1741895f65463 into 44103e89da4842b2ba32d1e620fd8e89(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:10,647 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:10,647 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/A, priority=13, startTime=1733771049802; duration=0sec 2024-12-09T19:04:10,647 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:10,647 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:04:10,655 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/46b145be82d947e8aa817fc8058c28eb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/46b145be82d947e8aa817fc8058c28eb 2024-12-09T19:04:10,660 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/C of 524e93dcb7c45b6722e1741895f65463 into 46b145be82d947e8aa817fc8058c28eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:10,661 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:10,661 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/C, priority=13, startTime=1733771049802; duration=0sec 2024-12-09T19:04:10,661 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:10,661 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:04:10,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,689 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/9b60a160392745af9b686490aa812498 2024-12-09T19:04:10,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/f19fd1c9143142a09ed8d4eafde6fdaa is 50, key is test_row_0/B:col10/1733771048171/Put/seqid=0 2024-12-09T19:04:10,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742005_1181 (size=12151) 2024-12-09T19:04:10,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:10,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771111035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:11,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771111035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771111044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771111054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:11,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/f19fd1c9143142a09ed8d4eafde6fdaa 2024-12-09T19:04:11,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a800c5ec3e784a51bebc2237fad5acbb is 50, key is test_row_0/C:col10/1733771048171/Put/seqid=0 2024-12-09T19:04:11,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742006_1182 (size=12151) 2024-12-09T19:04:11,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:11,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:11,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a800c5ec3e784a51bebc2237fad5acbb 2024-12-09T19:04:11,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/9b60a160392745af9b686490aa812498 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498 2024-12-09T19:04:11,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498, entries=200, sequenceid=233, filesize=38.8 K 2024-12-09T19:04:11,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/f19fd1c9143142a09ed8d4eafde6fdaa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f19fd1c9143142a09ed8d4eafde6fdaa 2024-12-09T19:04:11,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f19fd1c9143142a09ed8d4eafde6fdaa, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T19:04:11,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a800c5ec3e784a51bebc2237fad5acbb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a800c5ec3e784a51bebc2237fad5acbb 2024-12-09T19:04:11,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a800c5ec3e784a51bebc2237fad5acbb, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T19:04:11,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 524e93dcb7c45b6722e1741895f65463 in 1663ms, sequenceid=233, compaction requested=false 2024-12-09T19:04:11,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:11,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T19:04:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:11,589 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-09T19:04:11,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209418d4867ba9441e0bd14653fde06db83_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771049900/Put/seqid=0 2024-12-09T19:04:11,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742007_1183 (size=12304) 2024-12-09T19:04:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:12,003 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209418d4867ba9441e0bd14653fde06db83_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209418d4867ba9441e0bd14653fde06db83_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/43adb57b48a34805a506c9dd0bb34dcf, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/43adb57b48a34805a506c9dd0bb34dcf is 175, key is test_row_0/A:col10/1733771049900/Put/seqid=0 2024-12-09T19:04:12,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742008_1184 (size=31105) 2024-12-09T19:04:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:12,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:12,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771112069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771112071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771112072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771112072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771112174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771112184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771112184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771112184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771112354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,355 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 524e93dcb7c45b6722e1741895f65463 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771112384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771112394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771112395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771112395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/43adb57b48a34805a506c9dd0bb34dcf 2024-12-09T19:04:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/820bb8ee19b043b2b82ff2a5d784cded is 50, key is test_row_0/B:col10/1733771049900/Put/seqid=0 2024-12-09T19:04:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742009_1185 (size=12151) 2024-12-09T19:04:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771112694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771112704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771112704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771112704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:12,821 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/820bb8ee19b043b2b82ff2a5d784cded 2024-12-09T19:04:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/8e220fbe26334a01aee9559331db4067 is 50, key is test_row_0/C:col10/1733771049900/Put/seqid=0 2024-12-09T19:04:12,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742010_1186 (size=12151) 2024-12-09T19:04:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771113204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771113214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771113214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771113214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:13,233 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/8e220fbe26334a01aee9559331db4067 2024-12-09T19:04:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/43adb57b48a34805a506c9dd0bb34dcf as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf 2024-12-09T19:04:13,242 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf, entries=150, sequenceid=253, filesize=30.4 K 2024-12-09T19:04:13,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/820bb8ee19b043b2b82ff2a5d784cded as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/820bb8ee19b043b2b82ff2a5d784cded 2024-12-09T19:04:13,247 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/820bb8ee19b043b2b82ff2a5d784cded, entries=150, sequenceid=253, filesize=11.9 K 2024-12-09T19:04:13,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/8e220fbe26334a01aee9559331db4067 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/8e220fbe26334a01aee9559331db4067 2024-12-09T19:04:13,252 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/8e220fbe26334a01aee9559331db4067, entries=150, sequenceid=253, filesize=11.9 K 2024-12-09T19:04:13,253 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 524e93dcb7c45b6722e1741895f65463 in 1664ms, sequenceid=253, compaction requested=true 2024-12-09T19:04:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T19:04:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T19:04:13,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T19:04:13,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3470 sec 2024-12-09T19:04:13,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.3520 sec 2024-12-09T19:04:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T19:04:14,075 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:14,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-09T19:04:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:14,077 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:14,077 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:14,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:14,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-09T19:04:14,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c576fe4dc45442f0a582451514b142a8_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:14,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742011_1187 (size=14994) 2024-12-09T19:04:14,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771114239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771114239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771114241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771114240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771114344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771114354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771114354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771114354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:14,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771114554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771114564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771114564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771114564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:14,640 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c576fe4dc45442f0a582451514b142a8_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c576fe4dc45442f0a582451514b142a8_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:14,640 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0941383b1d43443a921a1742b516a796, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:14,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0941383b1d43443a921a1742b516a796 is 175, key is test_row_0/A:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:14,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742012_1188 (size=39949) 2024-12-09T19:04:14,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:14,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771114865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771114874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771114874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771114874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:14,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:14,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,052 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0941383b1d43443a921a1742b516a796 2024-12-09T19:04:15,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/f90e165d49154b2f8fca7bacc76585df is 50, key is test_row_0/B:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:15,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742013_1189 (size=12301) 2024-12-09T19:04:15,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/f90e165d49154b2f8fca7bacc76585df 2024-12-09T19:04:15,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/e58aca367fb84a7390cfe1cbd86f7bb3 is 50, key is test_row_0/C:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:15,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742014_1190 (size=12301) 2024-12-09T19:04:15,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:15,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771115374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771115384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771115384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771115384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:15,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:15,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/e58aca367fb84a7390cfe1cbd86f7bb3 2024-12-09T19:04:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0941383b1d43443a921a1742b516a796 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796 2024-12-09T19:04:15,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796, entries=200, sequenceid=274, filesize=39.0 K 2024-12-09T19:04:15,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/f90e165d49154b2f8fca7bacc76585df as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f90e165d49154b2f8fca7bacc76585df 2024-12-09T19:04:15,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f90e165d49154b2f8fca7bacc76585df, entries=150, sequenceid=274, filesize=12.0 K 2024-12-09T19:04:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/e58aca367fb84a7390cfe1cbd86f7bb3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/e58aca367fb84a7390cfe1cbd86f7bb3 2024-12-09T19:04:15,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/e58aca367fb84a7390cfe1cbd86f7bb3, entries=150, sequenceid=274, filesize=12.0 K 2024-12-09T19:04:15,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 524e93dcb7c45b6722e1741895f65463 in 1276ms, sequenceid=274, compaction requested=true 2024-12-09T19:04:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:15,493 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:04:15,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:15,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:15,494 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:04:15,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/B is initiating minor compaction (all files) 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/A is initiating minor compaction (all files) 2024-12-09T19:04:15,495 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/B in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,495 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/A in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,495 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/31d8b84cd1f44309957b4a8d8fe22679, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f19fd1c9143142a09ed8d4eafde6fdaa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/820bb8ee19b043b2b82ff2a5d784cded, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f90e165d49154b2f8fca7bacc76585df] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=48.1 K 2024-12-09T19:04:15,495 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/44103e89da4842b2ba32d1e620fd8e89, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=139.1 K 2024-12-09T19:04:15,495 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/44103e89da4842b2ba32d1e620fd8e89, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796] 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d8b84cd1f44309957b4a8d8fe22679, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733771047759 2024-12-09T19:04:15,495 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44103e89da4842b2ba32d1e620fd8e89, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733771047759 2024-12-09T19:04:15,496 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f19fd1c9143142a09ed8d4eafde6fdaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733771048171 2024-12-09T19:04:15,496 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b60a160392745af9b686490aa812498, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733771048171 2024-12-09T19:04:15,496 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 820bb8ee19b043b2b82ff2a5d784cded, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733771049888 2024-12-09T19:04:15,496 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43adb57b48a34805a506c9dd0bb34dcf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733771049888 2024-12-09T19:04:15,496 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f90e165d49154b2f8fca7bacc76585df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733771052069 2024-12-09T19:04:15,496 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0941383b1d43443a921a1742b516a796, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733771052069 2024-12-09T19:04:15,503 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:15,505 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:15,505 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/ca68f4aa3bdd464f9513e570dcb48f2f is 50, key is test_row_0/B:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:15,508 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209fce56b61b37641c182a009ad571b6cbf_524e93dcb7c45b6722e1741895f65463 store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:15,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742015_1191 (size=12949) 2024-12-09T19:04:15,510 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209fce56b61b37641c182a009ad571b6cbf_524e93dcb7c45b6722e1741895f65463, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:15,510 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fce56b61b37641c182a009ad571b6cbf_524e93dcb7c45b6722e1741895f65463 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:15,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742016_1192 (size=4469) 2024-12-09T19:04:15,516 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/ca68f4aa3bdd464f9513e570dcb48f2f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/ca68f4aa3bdd464f9513e570dcb48f2f 2024-12-09T19:04:15,517 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#A#compaction#157 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:15,517 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e1abc8a8bfd44c84a2d36edfe46ba1db is 175, key is test_row_0/A:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:15,521 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/B of 524e93dcb7c45b6722e1741895f65463 into ca68f4aa3bdd464f9513e570dcb48f2f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:15,521 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:15,521 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/B, priority=12, startTime=1733771055493; duration=0sec 2024-12-09T19:04:15,521 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:15,521 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:04:15,521 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:04:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742017_1193 (size=31903) 2024-12-09T19:04:15,523 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:04:15,523 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 524e93dcb7c45b6722e1741895f65463/C is initiating minor compaction (all files) 2024-12-09T19:04:15,523 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 524e93dcb7c45b6722e1741895f65463/C in TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,523 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/46b145be82d947e8aa817fc8058c28eb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a800c5ec3e784a51bebc2237fad5acbb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/8e220fbe26334a01aee9559331db4067, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/e58aca367fb84a7390cfe1cbd86f7bb3] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp, totalSize=48.1 K 2024-12-09T19:04:15,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 46b145be82d947e8aa817fc8058c28eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733771047759 2024-12-09T19:04:15,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a800c5ec3e784a51bebc2237fad5acbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733771048171 2024-12-09T19:04:15,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e220fbe26334a01aee9559331db4067, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733771049888 2024-12-09T19:04:15,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e58aca367fb84a7390cfe1cbd86f7bb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733771052069 2024-12-09T19:04:15,528 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/e1abc8a8bfd44c84a2d36edfe46ba1db as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e1abc8a8bfd44c84a2d36edfe46ba1db 2024-12-09T19:04:15,533 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/A of 524e93dcb7c45b6722e1741895f65463 into e1abc8a8bfd44c84a2d36edfe46ba1db(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:15,533 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:15,533 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/A, priority=12, startTime=1733771055493; duration=0sec 2024-12-09T19:04:15,533 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:15,533 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:04:15,534 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 524e93dcb7c45b6722e1741895f65463#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:15,535 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/3362936156ce49cfa1704027436afd38 is 50, key is test_row_0/C:col10/1733771052069/Put/seqid=0 2024-12-09T19:04:15,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742018_1194 (size=12949) 2024-12-09T19:04:15,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T19:04:15,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:15,604 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-09T19:04:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120909ccc7858c4c46bc98f0613e151f482c_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771054239/Put/seqid=0 2024-12-09T19:04:15,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742019_1195 (size=12454) 2024-12-09T19:04:15,946 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/3362936156ce49cfa1704027436afd38 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/3362936156ce49cfa1704027436afd38 2024-12-09T19:04:15,951 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 524e93dcb7c45b6722e1741895f65463/C of 524e93dcb7c45b6722e1741895f65463 into 3362936156ce49cfa1704027436afd38(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:15,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:15,951 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., storeName=524e93dcb7c45b6722e1741895f65463/C, priority=12, startTime=1733771055494; duration=0sec 2024-12-09T19:04:15,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:15,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:04:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:16,020 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120909ccc7858c4c46bc98f0613e151f482c_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120909ccc7858c4c46bc98f0613e151f482c_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:16,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0ab60792bd09456c8e74487f49ad4559, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:16,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0ab60792bd09456c8e74487f49ad4559 is 175, key is test_row_0/A:col10/1733771054239/Put/seqid=0 2024-12-09T19:04:16,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742020_1196 (size=31255) 2024-12-09T19:04:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:16,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. as already flushing 2024-12-09T19:04:16,425 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0ab60792bd09456c8e74487f49ad4559 2024-12-09T19:04:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/21090517d7d842229bf1a63ec83f1c3e is 50, key is test_row_0/B:col10/1733771054239/Put/seqid=0 2024-12-09T19:04:16,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771116428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771116431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742021_1197 (size=12301) 2024-12-09T19:04:16,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771116433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771116433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771116433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771116544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771116544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771116544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771116544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771116544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,635 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-12-09T19:04:16,635 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:16,635 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,635 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:16,636 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-12-09T19:04:16,636 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:16,636 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,636 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:16,637 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-12-09T19:04:16,637 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:16,637 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:16,637 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,638 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-12-09T19:04:16,639 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:16,639 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,639 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:16,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:16,699 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T19:04:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771116755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771116755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771116756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771116756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771116756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:16,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/21090517d7d842229bf1a63ec83f1c3e 2024-12-09T19:04:16,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a3c7689120124f76a5e6dccb43eb72c4 is 50, key is test_row_0/C:col10/1733771054239/Put/seqid=0 2024-12-09T19:04:16,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742022_1198 (size=12301) 2024-12-09T19:04:17,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:17,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58458 deadline: 1733771117065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58420 deadline: 1733771117065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58404 deadline: 1733771117065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58436 deadline: 1733771117066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1733771117065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:17,257 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a3c7689120124f76a5e6dccb43eb72c4 2024-12-09T19:04:17,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/0ab60792bd09456c8e74487f49ad4559 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0ab60792bd09456c8e74487f49ad4559 2024-12-09T19:04:17,276 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0ab60792bd09456c8e74487f49ad4559, entries=150, sequenceid=291, filesize=30.5 K 2024-12-09T19:04:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/21090517d7d842229bf1a63ec83f1c3e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/21090517d7d842229bf1a63ec83f1c3e 2024-12-09T19:04:17,283 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/21090517d7d842229bf1a63ec83f1c3e, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T19:04:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/a3c7689120124f76a5e6dccb43eb72c4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a3c7689120124f76a5e6dccb43eb72c4 2024-12-09T19:04:17,289 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a3c7689120124f76a5e6dccb43eb72c4, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T19:04:17,291 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for 524e93dcb7c45b6722e1741895f65463 in 1686ms, sequenceid=291, compaction requested=false 2024-12-09T19:04:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:17,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-09T19:04:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-09T19:04:17,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T19:04:17,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2150 sec 2024-12-09T19:04:17,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.2190 sec 2024-12-09T19:04:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:17,580 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:17,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:04:17,580 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-12-09T19:04:17,581 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:17,581 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,581 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-12-09T19:04:17,581 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,581 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-12-09T19:04:17,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:17,581 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:17,581 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,581 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:17,581 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,581 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:17,581 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:17,582 INFO [Thread-595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-595. 2024-12-09T19:04:17,583 DEBUG [Thread-595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:17,583 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,583 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:17,584 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-12-09T19:04:17,584 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:17,584 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,584 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:17,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:17,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209563c366a16f3425ba40cd5ac1e602bb8_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771057576/Put/seqid=0 2024-12-09T19:04:17,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742023_1199 (size=12454) 2024-12-09T19:04:17,996 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:18,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209563c366a16f3425ba40cd5ac1e602bb8_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209563c366a16f3425ba40cd5ac1e602bb8_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:18,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c4c5d23ebef64197bb7510846d6befa4, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:18,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c4c5d23ebef64197bb7510846d6befa4 is 175, key is test_row_0/A:col10/1733771057576/Put/seqid=0 2024-12-09T19:04:18,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742024_1200 (size=31255) 2024-12-09T19:04:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T19:04:18,246 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-09T19:04:18,246 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5149 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4907 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2174 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6520 rows 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2182 2024-12-09T19:04:18,247 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6542 rows 2024-12-09T19:04:18,247 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:04:18,247 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:04:18,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:18,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:18,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:18,250 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T19:04:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:18,257 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771058256"}]},"ts":"1733771058256"} 2024-12-09T19:04:18,259 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T19:04:18,259 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T19:04:18,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:04:18,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, UNASSIGN}] 2024-12-09T19:04:18,263 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, UNASSIGN 2024-12-09T19:04:18,263 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:18,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, UNASSIGN because future has completed 2024-12-09T19:04:18,265 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:04:18,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:04:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:18,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c4c5d23ebef64197bb7510846d6befa4 2024-12-09T19:04:18,418 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:18,419 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:04:18,419 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 524e93dcb7c45b6722e1741895f65463, disabling compactions & flushes 2024-12-09T19:04:18,420 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:18,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/d693d5f3473046dcbebf7815a7188ee2 is 50, key is test_row_0/B:col10/1733771057576/Put/seqid=0 2024-12-09T19:04:18,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742025_1201 (size=12301) 2024-12-09T19:04:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:18,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/d693d5f3473046dcbebf7815a7188ee2 2024-12-09T19:04:18,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/4ed774c25c964fbc90f641f1aacc509e is 50, key is test_row_0/C:col10/1733771057576/Put/seqid=0 2024-12-09T19:04:18,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742026_1202 (size=12301) 2024-12-09T19:04:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:19,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/4ed774c25c964fbc90f641f1aacc509e 2024-12-09T19:04:19,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/c4c5d23ebef64197bb7510846d6befa4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c4c5d23ebef64197bb7510846d6befa4 2024-12-09T19:04:19,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c4c5d23ebef64197bb7510846d6befa4, entries=150, sequenceid=319, filesize=30.5 K 2024-12-09T19:04:19,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/d693d5f3473046dcbebf7815a7188ee2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/d693d5f3473046dcbebf7815a7188ee2 2024-12-09T19:04:19,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/d693d5f3473046dcbebf7815a7188ee2, entries=150, sequenceid=319, filesize=12.0 K 2024-12-09T19:04:19,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/4ed774c25c964fbc90f641f1aacc509e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/4ed774c25c964fbc90f641f1aacc509e 2024-12-09T19:04:19,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/4ed774c25c964fbc90f641f1aacc509e, entries=150, sequenceid=319, filesize=12.0 K 2024-12-09T19:04:19,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=6.71 KB/6870 for 524e93dcb7c45b6722e1741895f65463 in 1705ms, sequenceid=319, compaction requested=true 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 524e93dcb7c45b6722e1741895f65463: 2024-12-09T19:04:19,285 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:19,285 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:19,285 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. after waiting 0 ms 2024-12-09T19:04:19,285 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 524e93dcb7c45b6722e1741895f65463:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:19,285 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 524e93dcb7c45b6722e1741895f65463 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-12-09T19:04:19,285 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. because compaction request was cancelled 2024-12-09T19:04:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:19,285 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. because compaction request was cancelled 2024-12-09T19:04:19,285 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:A 2024-12-09T19:04:19,285 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:B 2024-12-09T19:04:19,285 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. because compaction request was cancelled 2024-12-09T19:04:19,285 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 524e93dcb7c45b6722e1741895f65463:C 2024-12-09T19:04:19,291 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092eee03d226fb46608ff8be0b3405b254_524e93dcb7c45b6722e1741895f65463 is 50, key is test_row_0/A:col10/1733771057583/Put/seqid=0 2024-12-09T19:04:19,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742027_1203 (size=7374) 2024-12-09T19:04:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:19,696 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:19,707 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092eee03d226fb46608ff8be0b3405b254_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092eee03d226fb46608ff8be0b3405b254_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:19,708 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/3ed0ff6ee76940f887565050cb9c7ae8, store: [table=TestAcidGuarantees family=A region=524e93dcb7c45b6722e1741895f65463] 2024-12-09T19:04:19,709 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/3ed0ff6ee76940f887565050cb9c7ae8 is 175, key is test_row_0/A:col10/1733771057583/Put/seqid=0 2024-12-09T19:04:19,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742028_1204 (size=13865) 2024-12-09T19:04:20,116 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/3ed0ff6ee76940f887565050cb9c7ae8 2024-12-09T19:04:20,145 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/db353754e372457f9023f8cc8fb1a00d is 50, key is test_row_0/B:col10/1733771057583/Put/seqid=0 2024-12-09T19:04:20,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742029_1205 (size=7415) 2024-12-09T19:04:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:20,551 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/db353754e372457f9023f8cc8fb1a00d 2024-12-09T19:04:20,566 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/cb76ebd39912469ca7d8a2c1043f3f86 is 50, key is test_row_0/C:col10/1733771057583/Put/seqid=0 2024-12-09T19:04:20,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742030_1206 (size=7415) 2024-12-09T19:04:20,970 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/cb76ebd39912469ca7d8a2c1043f3f86 2024-12-09T19:04:20,977 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/A/3ed0ff6ee76940f887565050cb9c7ae8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/3ed0ff6ee76940f887565050cb9c7ae8 2024-12-09T19:04:20,983 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/3ed0ff6ee76940f887565050cb9c7ae8, entries=50, sequenceid=323, filesize=13.5 K 2024-12-09T19:04:20,985 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/B/db353754e372457f9023f8cc8fb1a00d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/db353754e372457f9023f8cc8fb1a00d 2024-12-09T19:04:20,991 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/db353754e372457f9023f8cc8fb1a00d, entries=50, sequenceid=323, filesize=7.2 K 2024-12-09T19:04:20,993 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/.tmp/C/cb76ebd39912469ca7d8a2c1043f3f86 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/cb76ebd39912469ca7d8a2c1043f3f86 2024-12-09T19:04:20,998 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/cb76ebd39912469ca7d8a2c1043f3f86, entries=50, sequenceid=323, filesize=7.2 K 2024-12-09T19:04:20,999 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 524e93dcb7c45b6722e1741895f65463 in 1714ms, sequenceid=323, compaction requested=true 2024-12-09T19:04:21,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/cfe27efa29544afba98d8ada080b453f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/4c407c659f324f6586211469e4589e8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/529aa535492e4e94b91c7f129990bfca, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/5e8de4d6c2ab472380841b702ab3a74c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/44103e89da4842b2ba32d1e620fd8e89, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796] to archive 2024-12-09T19:04:21,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:04:21,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/fbb7706248a9402f844a60e96f0305fc 2024-12-09T19:04:21,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0d05f9ab4aa24822929f05534b63ad63 2024-12-09T19:04:21,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e8264e10699f4a51bc120d79bd6040a0 2024-12-09T19:04:21,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/cfe27efa29544afba98d8ada080b453f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/cfe27efa29544afba98d8ada080b453f 2024-12-09T19:04:21,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/33c77b14f5054d02856533f85f38d6ad 2024-12-09T19:04:21,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/f4fd6764c99a46bf9d993020c7d55bbb 2024-12-09T19:04:21,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/4c407c659f324f6586211469e4589e8f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/4c407c659f324f6586211469e4589e8f 2024-12-09T19:04:21,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/36e41ffeee67493d8308f237d6ea15e8 2024-12-09T19:04:21,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/529aa535492e4e94b91c7f129990bfca to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/529aa535492e4e94b91c7f129990bfca 2024-12-09T19:04:21,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/56654be8d42147d6ab4791f18511deb7 2024-12-09T19:04:21,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/30a929afec5646fcbbceeb76efba97f9 2024-12-09T19:04:21,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/6b54e46a1aea498d9bac27de5548c593 2024-12-09T19:04:21,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/5e8de4d6c2ab472380841b702ab3a74c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/5e8de4d6c2ab472380841b702ab3a74c 2024-12-09T19:04:21,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e86256e1ac874f45a72c2a75caa5e1e9 2024-12-09T19:04:21,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c88abf11ad17423da9af82232046922d 2024-12-09T19:04:21,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/44103e89da4842b2ba32d1e620fd8e89 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/44103e89da4842b2ba32d1e620fd8e89 2024-12-09T19:04:21,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/9b60a160392745af9b686490aa812498 2024-12-09T19:04:21,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/43adb57b48a34805a506c9dd0bb34dcf 2024-12-09T19:04:21,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0941383b1d43443a921a1742b516a796 2024-12-09T19:04:21,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/c1781e3c31d741b59b3af791fe805c67, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/533dd61f3afc43ccab6d4076606918d8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/313e8b6cf9b64ffd9a5664cbfbc67988, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/fc3b9a1e28f049369f9892cf44a3e26a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c4fd106e92d48308a8a95aef43e1499, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/7132fa23a4fa4b19abc8091cb2686a1d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/93e3f3a643e84a218ddad9f767dd7453, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/28aa8848b5ca4bb59690e12e5b8066b0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/cd7c6cafe62c4d22a07c2194f29ac488, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/1605813e6b254b67b50a1c8afa9320ef, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/50380a3243ae41bfb754214c4448f4e2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5938079b3f874c409e9d0799d67867d9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/4944e9c3088b4eb58409607e3bfe1dba, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c747758a9b343f4b8232839a8f1023e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/31d8b84cd1f44309957b4a8d8fe22679, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/46cfa4e6c8c744fbad66a6c000ca3a8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f19fd1c9143142a09ed8d4eafde6fdaa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/820bb8ee19b043b2b82ff2a5d784cded, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f90e165d49154b2f8fca7bacc76585df] to archive 2024-12-09T19:04:21,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:04:21,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/c1781e3c31d741b59b3af791fe805c67 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/c1781e3c31d741b59b3af791fe805c67 2024-12-09T19:04:21,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/533dd61f3afc43ccab6d4076606918d8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/533dd61f3afc43ccab6d4076606918d8 2024-12-09T19:04:21,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/313e8b6cf9b64ffd9a5664cbfbc67988 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/313e8b6cf9b64ffd9a5664cbfbc67988 2024-12-09T19:04:21,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/fc3b9a1e28f049369f9892cf44a3e26a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/fc3b9a1e28f049369f9892cf44a3e26a 2024-12-09T19:04:21,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c4fd106e92d48308a8a95aef43e1499 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c4fd106e92d48308a8a95aef43e1499 2024-12-09T19:04:21,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/7132fa23a4fa4b19abc8091cb2686a1d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/7132fa23a4fa4b19abc8091cb2686a1d 2024-12-09T19:04:21,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/93e3f3a643e84a218ddad9f767dd7453 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/93e3f3a643e84a218ddad9f767dd7453 2024-12-09T19:04:21,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/28aa8848b5ca4bb59690e12e5b8066b0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/28aa8848b5ca4bb59690e12e5b8066b0 2024-12-09T19:04:21,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/cd7c6cafe62c4d22a07c2194f29ac488 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/cd7c6cafe62c4d22a07c2194f29ac488 2024-12-09T19:04:21,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/1605813e6b254b67b50a1c8afa9320ef to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/1605813e6b254b67b50a1c8afa9320ef 2024-12-09T19:04:21,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/50380a3243ae41bfb754214c4448f4e2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/50380a3243ae41bfb754214c4448f4e2 2024-12-09T19:04:21,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5938079b3f874c409e9d0799d67867d9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5938079b3f874c409e9d0799d67867d9 2024-12-09T19:04:21,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/4944e9c3088b4eb58409607e3bfe1dba to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/4944e9c3088b4eb58409607e3bfe1dba 2024-12-09T19:04:21,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c747758a9b343f4b8232839a8f1023e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/5c747758a9b343f4b8232839a8f1023e 2024-12-09T19:04:21,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/31d8b84cd1f44309957b4a8d8fe22679 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/31d8b84cd1f44309957b4a8d8fe22679 2024-12-09T19:04:21,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/46cfa4e6c8c744fbad66a6c000ca3a8f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/46cfa4e6c8c744fbad66a6c000ca3a8f 2024-12-09T19:04:21,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f19fd1c9143142a09ed8d4eafde6fdaa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f19fd1c9143142a09ed8d4eafde6fdaa 2024-12-09T19:04:21,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/820bb8ee19b043b2b82ff2a5d784cded to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/820bb8ee19b043b2b82ff2a5d784cded 2024-12-09T19:04:21,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f90e165d49154b2f8fca7bacc76585df to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/f90e165d49154b2f8fca7bacc76585df 2024-12-09T19:04:21,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/b2ee67c7a1004e379a8a7ef9db712814, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/feae470929ef498684bcc0aa5570fe26, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/98f749dce5e34aa597019702b842ec01, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/2526ca4fefd84be3836f18ef70a6693a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a7cca53341bb4fa4b137ca050c4e4d11, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/38d49387c1bb4aa2a71c4b107bbe8bb2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/81c2c3ec40a0495d8eed4787a1574816, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/bba82437df1443668f9e4eb8e8deabd4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a10b63b0df8949fbb717d0467a09df2b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1e914cd9e78c4c82bc10b76ce94e3f3c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/45b8fbd87b104aca803b704a8ac95c22, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1ffa1932e74b4e2ebc782eca5fef666e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/761c75c2a81b45df8198f8318fe18156, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1c094820b48a44e183de3683afc40e98, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/46b145be82d947e8aa817fc8058c28eb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/96c45f957cb2495e875c4cb8a6baf726, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a800c5ec3e784a51bebc2237fad5acbb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/8e220fbe26334a01aee9559331db4067, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/e58aca367fb84a7390cfe1cbd86f7bb3] to archive 2024-12-09T19:04:21,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:04:21,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/b2ee67c7a1004e379a8a7ef9db712814 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/b2ee67c7a1004e379a8a7ef9db712814 2024-12-09T19:04:21,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/feae470929ef498684bcc0aa5570fe26 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/feae470929ef498684bcc0aa5570fe26 2024-12-09T19:04:21,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/98f749dce5e34aa597019702b842ec01 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/98f749dce5e34aa597019702b842ec01 2024-12-09T19:04:21,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/2526ca4fefd84be3836f18ef70a6693a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/2526ca4fefd84be3836f18ef70a6693a 2024-12-09T19:04:21,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a7cca53341bb4fa4b137ca050c4e4d11 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a7cca53341bb4fa4b137ca050c4e4d11 2024-12-09T19:04:21,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/38d49387c1bb4aa2a71c4b107bbe8bb2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/38d49387c1bb4aa2a71c4b107bbe8bb2 2024-12-09T19:04:21,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/81c2c3ec40a0495d8eed4787a1574816 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/81c2c3ec40a0495d8eed4787a1574816 2024-12-09T19:04:21,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/bba82437df1443668f9e4eb8e8deabd4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/bba82437df1443668f9e4eb8e8deabd4 2024-12-09T19:04:21,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a10b63b0df8949fbb717d0467a09df2b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a10b63b0df8949fbb717d0467a09df2b 2024-12-09T19:04:21,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1e914cd9e78c4c82bc10b76ce94e3f3c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1e914cd9e78c4c82bc10b76ce94e3f3c 2024-12-09T19:04:21,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/45b8fbd87b104aca803b704a8ac95c22 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/45b8fbd87b104aca803b704a8ac95c22 2024-12-09T19:04:21,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1ffa1932e74b4e2ebc782eca5fef666e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1ffa1932e74b4e2ebc782eca5fef666e 2024-12-09T19:04:21,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/761c75c2a81b45df8198f8318fe18156 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/761c75c2a81b45df8198f8318fe18156 2024-12-09T19:04:21,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1c094820b48a44e183de3683afc40e98 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/1c094820b48a44e183de3683afc40e98 2024-12-09T19:04:21,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/46b145be82d947e8aa817fc8058c28eb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/46b145be82d947e8aa817fc8058c28eb 2024-12-09T19:04:21,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/96c45f957cb2495e875c4cb8a6baf726 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/96c45f957cb2495e875c4cb8a6baf726 2024-12-09T19:04:21,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a800c5ec3e784a51bebc2237fad5acbb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a800c5ec3e784a51bebc2237fad5acbb 2024-12-09T19:04:21,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/8e220fbe26334a01aee9559331db4067 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/8e220fbe26334a01aee9559331db4067 2024-12-09T19:04:21,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/e58aca367fb84a7390cfe1cbd86f7bb3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/e58aca367fb84a7390cfe1cbd86f7bb3 2024-12-09T19:04:21,062 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-12-09T19:04:21,062 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. 2024-12-09T19:04:21,062 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 524e93dcb7c45b6722e1741895f65463: Waiting for close lock at 1733771058419Running coprocessor pre-close hooks at 1733771058419Disabling compacts and flushes for region at 1733771058419Disabling writes for close at 1733771059285 (+866 ms)Obtaining lock to block concurrent updates at 1733771059285Preparing flush snapshotting stores in 524e93dcb7c45b6722e1741895f65463 at 1733771059285Finished memstore snapshotting TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1733771059285Flushing stores of TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463. at 1733771059286 (+1 ms)Flushing 524e93dcb7c45b6722e1741895f65463/A: creating writer at 1733771059286Flushing mob file 524e93dcb7c45b6722e1741895f65463/A: appending metadata at 1733771059291 (+5 ms)Flushing mob file 524e93dcb7c45b6722e1741895f65463/A: closing flushed file at 1733771059291Flushing 524e93dcb7c45b6722e1741895f65463/A: appending metadata at 1733771059708 (+417 ms)Flushing 524e93dcb7c45b6722e1741895f65463/A: closing flushed file at 1733771059708Flushing 524e93dcb7c45b6722e1741895f65463/B: creating writer at 1733771060142 (+434 ms)Flushing 524e93dcb7c45b6722e1741895f65463/B: appending metadata at 1733771060145 (+3 ms)Flushing 524e93dcb7c45b6722e1741895f65463/B: closing flushed file at 1733771060145Flushing 524e93dcb7c45b6722e1741895f65463/C: creating writer at 1733771060562 (+417 ms)Flushing 524e93dcb7c45b6722e1741895f65463/C: appending metadata at 1733771060565 (+3 ms)Flushing 524e93dcb7c45b6722e1741895f65463/C: closing flushed file at 1733771060565Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53f9cd96: reopening flushed file at 1733771060976 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@299f3cdc: reopening flushed file at 1733771060984 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c10bc5a: reopening flushed file at 1733771060992 (+8 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 524e93dcb7c45b6722e1741895f65463 in 1714ms, sequenceid=323, compaction requested=true at 1733771060999 (+7 ms)Writing region close event to WAL at 1733771061059 (+60 ms)Running coprocessor post-close hooks at 1733771061062 (+3 ms)Closed at 1733771061062 2024-12-09T19:04:21,064 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:21,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=524e93dcb7c45b6722e1741895f65463, regionState=CLOSED 2024-12-09T19:04:21,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:04:21,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-09T19:04:21,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 524e93dcb7c45b6722e1741895f65463, server=a437f8b9ba7d,46189,1733770998237 in 2.8010 sec 2024-12-09T19:04:21,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-09T19:04:21,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=524e93dcb7c45b6722e1741895f65463, UNASSIGN in 2.8060 sec 2024-12-09T19:04:21,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T19:04:21,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8080 sec 2024-12-09T19:04:21,072 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771061071"}]},"ts":"1733771061071"} 2024-12-09T19:04:21,073 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T19:04:21,073 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T19:04:21,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8230 sec 2024-12-09T19:04:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T19:04:22,436 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:22,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T19:04:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,442 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T19:04:22,445 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,448 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,451 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/recovered.edits] 2024-12-09T19:04:22,454 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0ab60792bd09456c8e74487f49ad4559 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/0ab60792bd09456c8e74487f49ad4559 2024-12-09T19:04:22,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/3ed0ff6ee76940f887565050cb9c7ae8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/3ed0ff6ee76940f887565050cb9c7ae8 2024-12-09T19:04:22,457 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c4c5d23ebef64197bb7510846d6befa4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/c4c5d23ebef64197bb7510846d6befa4 2024-12-09T19:04:22,458 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e1abc8a8bfd44c84a2d36edfe46ba1db to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/A/e1abc8a8bfd44c84a2d36edfe46ba1db 2024-12-09T19:04:22,461 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/21090517d7d842229bf1a63ec83f1c3e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/21090517d7d842229bf1a63ec83f1c3e 2024-12-09T19:04:22,461 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/ca68f4aa3bdd464f9513e570dcb48f2f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/ca68f4aa3bdd464f9513e570dcb48f2f 2024-12-09T19:04:22,462 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/d693d5f3473046dcbebf7815a7188ee2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/d693d5f3473046dcbebf7815a7188ee2 2024-12-09T19:04:22,463 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/db353754e372457f9023f8cc8fb1a00d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/B/db353754e372457f9023f8cc8fb1a00d 2024-12-09T19:04:22,465 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/3362936156ce49cfa1704027436afd38 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/3362936156ce49cfa1704027436afd38 2024-12-09T19:04:22,466 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/4ed774c25c964fbc90f641f1aacc509e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/4ed774c25c964fbc90f641f1aacc509e 2024-12-09T19:04:22,466 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a3c7689120124f76a5e6dccb43eb72c4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/a3c7689120124f76a5e6dccb43eb72c4 2024-12-09T19:04:22,467 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/cb76ebd39912469ca7d8a2c1043f3f86 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/C/cb76ebd39912469ca7d8a2c1043f3f86 2024-12-09T19:04:22,469 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/recovered.edits/326.seqid to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463/recovered.edits/326.seqid 2024-12-09T19:04:22,470 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,470 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T19:04:22,470 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T19:04:22,471 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T19:04:22,474 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120909ccc7858c4c46bc98f0613e151f482c_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120909ccc7858c4c46bc98f0613e151f482c_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,475 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209160a9ea8fc8e464ea9499ed45985652d_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209160a9ea8fc8e464ea9499ed45985652d_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091896e9bf80c045509b7309a600f54170_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091896e9bf80c045509b7309a600f54170_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,477 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092eee03d226fb46608ff8be0b3405b254_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092eee03d226fb46608ff8be0b3405b254_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,478 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209418d4867ba9441e0bd14653fde06db83_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209418d4867ba9441e0bd14653fde06db83_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,479 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094fcc2d6791fa42cfba7a195899a4ccab_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094fcc2d6791fa42cfba7a195899a4ccab_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,480 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095564a290848f460e9a5bdd1c021caa2e_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095564a290848f460e9a5bdd1c021caa2e_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,481 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209563c366a16f3425ba40cd5ac1e602bb8_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209563c366a16f3425ba40cd5ac1e602bb8_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,482 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d7c6f95ef0b489da7abf8c1c455566d_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d7c6f95ef0b489da7abf8c1c455566d_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,483 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120967287163119147929f5b728df9317bec_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120967287163119147929f5b728df9317bec_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,485 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097ae2b9eb66a44a83aff58c87ec7876f3_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097ae2b9eb66a44a83aff58c87ec7876f3_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120980e6ac915e2047ce91068d48779869be_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120980e6ac915e2047ce91068d48779869be_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,487 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099c5a30ead234438d8c64e9334ab7718b_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099c5a30ead234438d8c64e9334ab7718b_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,489 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab548ed2ebe04f979d25bf8f6a3cc15c_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ab548ed2ebe04f979d25bf8f6a3cc15c_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,490 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8902a253954450ab1ddd4d514ef50b6_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8902a253954450ab1ddd4d514ef50b6_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,491 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c576fe4dc45442f0a582451514b142a8_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c576fe4dc45442f0a582451514b142a8_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,492 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f5848bb0cf1c458f8b14cb80e62de1d4_524e93dcb7c45b6722e1741895f65463 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f5848bb0cf1c458f8b14cb80e62de1d4_524e93dcb7c45b6722e1741895f65463 2024-12-09T19:04:22,493 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T19:04:22,495 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,497 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T19:04:22,499 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T19:04:22,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T19:04:22,500 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733771062500"}]},"ts":"9223372036854775807"} 2024-12-09T19:04:22,502 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T19:04:22,502 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 524e93dcb7c45b6722e1741895f65463, NAME => 'TestAcidGuarantees,,1733771033443.524e93dcb7c45b6722e1741895f65463.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T19:04:22,502 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T19:04:22,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733771062502"}]},"ts":"9223372036854775807"} 2024-12-09T19:04:22,504 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T19:04:22,505 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-12-09T19:04:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T19:04:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T19:04:22,555 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:22,562 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1672360030_22 at /127.0.0.1:46066 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2004503095_22 at /127.0.0.1:46050 [Waiting for operation #341] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1672360030_22 at /127.0.0.1:45896 [Waiting for operation #385] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2004503095_22 at /127.0.0.1:45814 [Waiting for operation #419] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 225) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2229 (was 2341) 2024-12-09T19:04:22,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=2229 2024-12-09T19:04:22,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:04:22,570 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:04:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:22,572 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T19:04:22,572 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:22,572 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-09T19:04:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T19:04:22,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T19:04:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742031_1207 (size=959) 2024-12-09T19:04:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T19:04:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T19:04:22,980 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:04:22,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742032_1208 (size=53) 2024-12-09T19:04:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T19:04:23,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:04:23,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f95442f56e862cbd2c6f1b449c5bb8c9, disabling compactions & flushes 2024-12-09T19:04:23,388 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. after waiting 0 ms 2024-12-09T19:04:23,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,388 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,389 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f95442f56e862cbd2c6f1b449c5bb8c9: Waiting for close lock at 1733771063388Disabling compacts and flushes for region at 1733771063388Disabling writes for close at 1733771063388Writing region close event to WAL at 1733771063388Closed at 1733771063388 2024-12-09T19:04:23,390 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T19:04:23,390 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733771063390"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733771063390"}]},"ts":"1733771063390"} 2024-12-09T19:04:23,392 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T19:04:23,392 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T19:04:23,393 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771063392"}]},"ts":"1733771063392"} 2024-12-09T19:04:23,394 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T19:04:23,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, ASSIGN}] 2024-12-09T19:04:23,395 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, ASSIGN 2024-12-09T19:04:23,396 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=false 2024-12-09T19:04:23,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f95442f56e862cbd2c6f1b449c5bb8c9, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:23,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, ASSIGN because future has completed 2024-12-09T19:04:23,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:04:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T19:04:23,709 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,709 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:04:23,709 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,709 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:04:23,709 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,709 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,710 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,712 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f95442f56e862cbd2c6f1b449c5bb8c9 columnFamilyName A 2024-12-09T19:04:23,712 DEBUG [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:23,712 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] regionserver.HStore(327): Store=f95442f56e862cbd2c6f1b449c5bb8c9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:23,712 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,714 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f95442f56e862cbd2c6f1b449c5bb8c9 columnFamilyName B 2024-12-09T19:04:23,714 DEBUG [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:23,714 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] regionserver.HStore(327): Store=f95442f56e862cbd2c6f1b449c5bb8c9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:23,714 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,715 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f95442f56e862cbd2c6f1b449c5bb8c9 columnFamilyName C 2024-12-09T19:04:23,715 DEBUG [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:23,716 INFO [StoreOpener-f95442f56e862cbd2c6f1b449c5bb8c9-1 {}] regionserver.HStore(327): Store=f95442f56e862cbd2c6f1b449c5bb8c9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:23,716 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,717 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,717 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,719 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,719 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,719 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:04:23,721 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,723 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:04:23,724 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened f95442f56e862cbd2c6f1b449c5bb8c9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70711742, jitterRate=0.053687065839767456}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:04:23,724 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:23,725 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for f95442f56e862cbd2c6f1b449c5bb8c9: Running coprocessor pre-open hook at 1733771063709Writing region info on filesystem at 1733771063709Initializing all the Stores at 1733771063710 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771063710Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771063710Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771063710Cleaning up temporary data from old regions at 1733771063719 (+9 ms)Running coprocessor post-open hooks at 1733771063724 (+5 ms)Region opened successfully at 1733771063725 (+1 ms) 2024-12-09T19:04:23,726 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., pid=61, masterSystemTime=1733771063706 2024-12-09T19:04:23,728 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,728 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:23,728 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f95442f56e862cbd2c6f1b449c5bb8c9, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:23,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:04:23,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T19:04:23,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 in 177 msec 2024-12-09T19:04:23,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-09T19:04:23,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, ASSIGN in 337 msec 2024-12-09T19:04:23,733 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T19:04:23,733 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771063733"}]},"ts":"1733771063733"} 2024-12-09T19:04:23,735 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T19:04:23,735 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T19:04:23,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-12-09T19:04:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-09T19:04:24,715 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:24,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723be589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3903852d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,723 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:04:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:04:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7082e236 2024-12-09T19:04:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:04:24,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:04:24,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56bc88e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e91f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,732 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683cacf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,734 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2126e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117f4bd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3349b350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,741 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1289ca42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3bb949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111e7d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109b24a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20dabf12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d00163e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bab203e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da93613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,755 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15377b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c364611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10639e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cbe636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:24,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:24,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0d3437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:24,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:24,761 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8cc018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:24,762 DEBUG [Thread-996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,762 DEBUG [Thread-1003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,762 DEBUG [Thread-1005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:24,763 DEBUG [Thread-1009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 DEBUG [Thread-1011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 DEBUG [Thread-994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 DEBUG [Thread-1000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,763 DEBUG [Thread-998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 DEBUG [Thread-1007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-09T19:04:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,764 DEBUG [Thread-992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,764 DEBUG [Thread-994 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,765 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:24,765 DEBUG [Thread-994 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:24,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:24,766 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:24,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:04:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:24,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:24,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:04:24,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/af94ab8912ed420fa7781d6856385f40 is 50, key is test_row_0/A:col10/1733771064773/Put/seqid=0 2024-12-09T19:04:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771124810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771124812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771124814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742033_1209 (size=12001) 2024-12-09T19:04:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771124816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771124816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:24,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:24,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771124925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771124925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771124925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771124925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771124925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771125135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771125135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771125135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771125135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771125135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/af94ab8912ed420fa7781d6856385f40 2024-12-09T19:04:25,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/fa7465c7a5da490c9f4c9a44b36ba400 is 50, key is test_row_0/B:col10/1733771064773/Put/seqid=0 2024-12-09T19:04:25,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742034_1210 (size=12001) 2024-12-09T19:04:25,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771125444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771125444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771125444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771125445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771125445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/fa7465c7a5da490c9f4c9a44b36ba400 2024-12-09T19:04:25,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/c25523d6b05c4616a5781470cabf7b25 is 50, key is test_row_0/C:col10/1733771064773/Put/seqid=0 2024-12-09T19:04:25,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742035_1211 (size=12001) 2024-12-09T19:04:25,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771125954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771125955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771125955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771125955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771125955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:25,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:25,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:26,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/c25523d6b05c4616a5781470cabf7b25 2024-12-09T19:04:26,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/af94ab8912ed420fa7781d6856385f40 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/af94ab8912ed420fa7781d6856385f40 2024-12-09T19:04:26,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/af94ab8912ed420fa7781d6856385f40, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T19:04:26,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/fa7465c7a5da490c9f4c9a44b36ba400 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/fa7465c7a5da490c9f4c9a44b36ba400 2024-12-09T19:04:26,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/fa7465c7a5da490c9f4c9a44b36ba400, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T19:04:26,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/c25523d6b05c4616a5781470cabf7b25 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/c25523d6b05c4616a5781470cabf7b25 2024-12-09T19:04:26,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/c25523d6b05c4616a5781470cabf7b25, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T19:04:26,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1319ms, sequenceid=13, compaction requested=false 2024-12-09T19:04:26,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:26,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-09T19:04:26,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:26,146 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:04:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/81cd170175b241a5a381e41bf82b8681 is 50, key is test_row_0/A:col10/1733771064814/Put/seqid=0 2024-12-09T19:04:26,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742036_1212 (size=12001) 2024-12-09T19:04:26,556 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/81cd170175b241a5a381e41bf82b8681 2024-12-09T19:04:26,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/1686c3da175b4dd1840b01278d5c1e1e is 50, key is test_row_0/B:col10/1733771064814/Put/seqid=0 2024-12-09T19:04:26,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742037_1213 (size=12001) 2024-12-09T19:04:26,567 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/1686c3da175b4dd1840b01278d5c1e1e 2024-12-09T19:04:26,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/8b293ed07e724524b858eda58b8db4d2 is 50, key is test_row_0/C:col10/1733771064814/Put/seqid=0 2024-12-09T19:04:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742038_1214 (size=12001) 2024-12-09T19:04:26,584 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:04:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:26,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771126971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771126971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771126971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771126974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771126974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:26,978 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:26,983 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/8b293ed07e724524b858eda58b8db4d2 2024-12-09T19:04:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/81cd170175b241a5a381e41bf82b8681 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/81cd170175b241a5a381e41bf82b8681 2024-12-09T19:04:26,991 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/81cd170175b241a5a381e41bf82b8681, entries=150, sequenceid=37, filesize=11.7 K 2024-12-09T19:04:26,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/1686c3da175b4dd1840b01278d5c1e1e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1686c3da175b4dd1840b01278d5c1e1e 2024-12-09T19:04:26,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1686c3da175b4dd1840b01278d5c1e1e, entries=150, sequenceid=37, filesize=11.7 K 2024-12-09T19:04:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/8b293ed07e724524b858eda58b8db4d2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8b293ed07e724524b858eda58b8db4d2 2024-12-09T19:04:27,001 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8b293ed07e724524b858eda58b8db4d2, entries=150, sequenceid=37, filesize=11.7 K 2024-12-09T19:04:27,002 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f95442f56e862cbd2c6f1b449c5bb8c9 in 857ms, sequenceid=37, compaction requested=false 2024-12-09T19:04:27,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:27,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:27,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-09T19:04:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-09T19:04:27,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-09T19:04:27,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-12-09T19:04:27,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.2420 sec 2024-12-09T19:04:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:27,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:27,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:04:27,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/32e76503a4f6440eb8e070f29b3c4102 is 50, key is test_row_0/A:col10/1733771067087/Put/seqid=0 2024-12-09T19:04:27,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742039_1215 (size=12001) 2024-12-09T19:04:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771127106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771127108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771127109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771127109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771127215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771127215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771127215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771127215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771127425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771127425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771127425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771127425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/32e76503a4f6440eb8e070f29b3c4102 2024-12-09T19:04:27,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/e4ac258c6e0446cfb78217ee12d9f316 is 50, key is test_row_0/B:col10/1733771067087/Put/seqid=0 2024-12-09T19:04:27,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742040_1216 (size=12001) 2024-12-09T19:04:27,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771127734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771127734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771127734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771127735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:27,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/e4ac258c6e0446cfb78217ee12d9f316 2024-12-09T19:04:27,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/595a4df6b81a446790ffa26437f53da6 is 50, key is test_row_0/C:col10/1733771067087/Put/seqid=0 2024-12-09T19:04:27,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742041_1217 (size=12001) 2024-12-09T19:04:28,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771128244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:28,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771128244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:28,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771128245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:28,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771128245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:28,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/595a4df6b81a446790ffa26437f53da6 2024-12-09T19:04:28,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/32e76503a4f6440eb8e070f29b3c4102 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/32e76503a4f6440eb8e070f29b3c4102 2024-12-09T19:04:28,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/32e76503a4f6440eb8e070f29b3c4102, entries=150, sequenceid=53, filesize=11.7 K 2024-12-09T19:04:28,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/e4ac258c6e0446cfb78217ee12d9f316 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4ac258c6e0446cfb78217ee12d9f316 2024-12-09T19:04:28,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4ac258c6e0446cfb78217ee12d9f316, entries=150, sequenceid=53, filesize=11.7 K 2024-12-09T19:04:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/595a4df6b81a446790ffa26437f53da6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/595a4df6b81a446790ffa26437f53da6 2024-12-09T19:04:28,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/595a4df6b81a446790ffa26437f53da6, entries=150, sequenceid=53, filesize=11.7 K 2024-12-09T19:04:28,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1250ms, sequenceid=53, compaction requested=true 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:28,337 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:28,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:28,337 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:28,338 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:28,338 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:28,338 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:28,338 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:28,339 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:28,339 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:28,339 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/fa7465c7a5da490c9f4c9a44b36ba400, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1686c3da175b4dd1840b01278d5c1e1e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4ac258c6e0446cfb78217ee12d9f316] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.2 K 2024-12-09T19:04:28,339 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/af94ab8912ed420fa7781d6856385f40, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/81cd170175b241a5a381e41bf82b8681, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/32e76503a4f6440eb8e070f29b3c4102] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.2 K 2024-12-09T19:04:28,339 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7465c7a5da490c9f4c9a44b36ba400, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733771064773 2024-12-09T19:04:28,339 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting af94ab8912ed420fa7781d6856385f40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733771064773 2024-12-09T19:04:28,340 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81cd170175b241a5a381e41bf82b8681, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733771064810 2024-12-09T19:04:28,340 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1686c3da175b4dd1840b01278d5c1e1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733771064810 2024-12-09T19:04:28,340 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ac258c6e0446cfb78217ee12d9f316, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733771066970 2024-12-09T19:04:28,340 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e76503a4f6440eb8e070f29b3c4102, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733771066970 2024-12-09T19:04:28,349 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:28,350 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/d91bcb7329c548a3ac7a6328bb1eeadf is 50, key is test_row_0/B:col10/1733771067087/Put/seqid=0 2024-12-09T19:04:28,357 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:28,357 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/00d0787ea1ab40619e8710ae3fc8ce33 is 50, key is test_row_0/A:col10/1733771067087/Put/seqid=0 2024-12-09T19:04:28,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742042_1218 (size=12104) 2024-12-09T19:04:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742043_1219 (size=12104) 2024-12-09T19:04:28,403 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/00d0787ea1ab40619e8710ae3fc8ce33 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/00d0787ea1ab40619e8710ae3fc8ce33 2024-12-09T19:04:28,413 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 00d0787ea1ab40619e8710ae3fc8ce33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:28,413 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:28,413 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771068337; duration=0sec 2024-12-09T19:04:28,413 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:28,413 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:28,413 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:28,414 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:28,414 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:28,414 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:28,415 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/c25523d6b05c4616a5781470cabf7b25, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8b293ed07e724524b858eda58b8db4d2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/595a4df6b81a446790ffa26437f53da6] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.2 K 2024-12-09T19:04:28,415 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25523d6b05c4616a5781470cabf7b25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733771064773 2024-12-09T19:04:28,415 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b293ed07e724524b858eda58b8db4d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733771064810 2024-12-09T19:04:28,416 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595a4df6b81a446790ffa26437f53da6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733771066970 2024-12-09T19:04:28,423 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:28,424 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/6ff6f1be9a514c50bc856ce60f049c98 is 50, key is test_row_0/C:col10/1733771067087/Put/seqid=0 2024-12-09T19:04:28,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742044_1220 (size=12104) 2024-12-09T19:04:28,799 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/d91bcb7329c548a3ac7a6328bb1eeadf as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/d91bcb7329c548a3ac7a6328bb1eeadf 2024-12-09T19:04:28,808 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into d91bcb7329c548a3ac7a6328bb1eeadf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:28,808 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:28,808 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771068337; duration=0sec 2024-12-09T19:04:28,808 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:28,808 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:28,838 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/6ff6f1be9a514c50bc856ce60f049c98 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6ff6f1be9a514c50bc856ce60f049c98 2024-12-09T19:04:28,844 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into 6ff6f1be9a514c50bc856ce60f049c98(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:28,844 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:28,844 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771068337; duration=0sec 2024-12-09T19:04:28,844 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:28,844 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T19:04:28,934 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:28,935 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-09T19:04:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T19:04:28,939 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:28,939 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:28,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:28,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-09T19:04:29,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/07f4375a5eae4cf094535d1096a2f709 is 50, key is test_row_0/A:col10/1733771067100/Put/seqid=0 2024-12-09T19:04:29,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742045_1221 (size=12001) 2024-12-09T19:04:29,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/07f4375a5eae4cf094535d1096a2f709 2024-12-09T19:04:29,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/76d9df0472b04488a77282192004b6b2 is 50, key is test_row_0/B:col10/1733771067100/Put/seqid=0 2024-12-09T19:04:29,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742046_1222 (size=12001) 2024-12-09T19:04:29,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/76d9df0472b04488a77282192004b6b2 2024-12-09T19:04:29,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/6799a237693e47ec9563703c881a1637 is 50, key is test_row_0/C:col10/1733771067100/Put/seqid=0 2024-12-09T19:04:29,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771129034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742047_1223 (size=12001) 2024-12-09T19:04:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T19:04:29,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T19:04:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771129144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T19:04:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T19:04:29,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771129255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771129255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,256 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:29,257 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771129264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771129264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,266 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:29,266 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771129354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T19:04:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:29,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/6799a237693e47ec9563703c881a1637 2024-12-09T19:04:29,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/07f4375a5eae4cf094535d1096a2f709 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/07f4375a5eae4cf094535d1096a2f709 2024-12-09T19:04:29,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/07f4375a5eae4cf094535d1096a2f709, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:04:29,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/76d9df0472b04488a77282192004b6b2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/76d9df0472b04488a77282192004b6b2 2024-12-09T19:04:29,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/76d9df0472b04488a77282192004b6b2, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:04:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/6799a237693e47ec9563703c881a1637 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6799a237693e47ec9563703c881a1637 2024-12-09T19:04:29,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6799a237693e47ec9563703c881a1637, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:04:29,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f95442f56e862cbd2c6f1b449c5bb8c9 in 465ms, sequenceid=78, compaction requested=false 2024-12-09T19:04:29,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:29,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-09T19:04:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:29,550 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-09T19:04:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/2a2fb6687118461a90b91c63bfc335b6 is 50, key is test_row_0/A:col10/1733771069001/Put/seqid=0 2024-12-09T19:04:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T19:04:29,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742048_1224 (size=12001) 2024-12-09T19:04:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:29,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771129738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771129845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:29,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/2a2fb6687118461a90b91c63bfc335b6 2024-12-09T19:04:29,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/83eb7da0ca0c4340bf24b80a6af1640b is 50, key is test_row_0/B:col10/1733771069001/Put/seqid=0 2024-12-09T19:04:29,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742049_1225 (size=12001) 2024-12-09T19:04:29,984 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/83eb7da0ca0c4340bf24b80a6af1640b 2024-12-09T19:04:29,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1d5d259866744d0cab6d13cde0e44a98 is 50, key is test_row_0/C:col10/1733771069001/Put/seqid=0 2024-12-09T19:04:29,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742050_1226 (size=12001) 2024-12-09T19:04:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771130054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T19:04:30,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771130364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:30,397 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1d5d259866744d0cab6d13cde0e44a98 2024-12-09T19:04:30,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/2a2fb6687118461a90b91c63bfc335b6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2a2fb6687118461a90b91c63bfc335b6 2024-12-09T19:04:30,429 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2a2fb6687118461a90b91c63bfc335b6, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:04:30,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/83eb7da0ca0c4340bf24b80a6af1640b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/83eb7da0ca0c4340bf24b80a6af1640b 2024-12-09T19:04:30,434 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/83eb7da0ca0c4340bf24b80a6af1640b, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:04:30,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1d5d259866744d0cab6d13cde0e44a98 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1d5d259866744d0cab6d13cde0e44a98 2024-12-09T19:04:30,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1d5d259866744d0cab6d13cde0e44a98, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:04:30,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f95442f56e862cbd2c6f1b449c5bb8c9 in 889ms, sequenceid=92, compaction requested=true 2024-12-09T19:04:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:30,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:30,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-09T19:04:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-09T19:04:30,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T19:04:30,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-12-09T19:04:30,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5080 sec 2024-12-09T19:04:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:30,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:30,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:04:30,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/461173547c544569a815e74dea1d44f4 is 50, key is test_row_0/A:col10/1733771069726/Put/seqid=0 2024-12-09T19:04:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742051_1227 (size=12001) 2024-12-09T19:04:30,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771130914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771131025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-09T19:04:31,085 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:31,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T19:04:31,087 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T19:04:31,087 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:31,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T19:04:31,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771131235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771131264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,266 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771131274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,276 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/461173547c544569a815e74dea1d44f4 2024-12-09T19:04:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771131284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,287 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771131295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,297 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/22b158644f6044f8bf52cacbc58690fb is 50, key is test_row_0/B:col10/1733771069726/Put/seqid=0 2024-12-09T19:04:31,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742052_1228 (size=12001) 2024-12-09T19:04:31,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T19:04:31,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:31,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771131545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/22b158644f6044f8bf52cacbc58690fb 2024-12-09T19:04:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T19:04:31,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/598d09c3686044f79e7d7d67c483fc83 is 50, key is test_row_0/C:col10/1733771069726/Put/seqid=0 2024-12-09T19:04:31,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742053_1229 (size=12001) 2024-12-09T19:04:31,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:31,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:31,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:32,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:32,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:32,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:32,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:32,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:32,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771132054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:32,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/598d09c3686044f79e7d7d67c483fc83 2024-12-09T19:04:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/461173547c544569a815e74dea1d44f4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/461173547c544569a815e74dea1d44f4 2024-12-09T19:04:32,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/461173547c544569a815e74dea1d44f4, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:04:32,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/22b158644f6044f8bf52cacbc58690fb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/22b158644f6044f8bf52cacbc58690fb 2024-12-09T19:04:32,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/22b158644f6044f8bf52cacbc58690fb, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:04:32,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/598d09c3686044f79e7d7d67c483fc83 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/598d09c3686044f79e7d7d67c483fc83 2024-12-09T19:04:32,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/598d09c3686044f79e7d7d67c483fc83, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:04:32,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1269ms, sequenceid=115, compaction requested=true 2024-12-09T19:04:32,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:32,147 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:04:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:32,147 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:04:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:32,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:32,148 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:04:32,148 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:04:32,148 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:32,148 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:32,148 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:32,148 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:32,148 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/d91bcb7329c548a3ac7a6328bb1eeadf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/76d9df0472b04488a77282192004b6b2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/83eb7da0ca0c4340bf24b80a6af1640b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/22b158644f6044f8bf52cacbc58690fb] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=47.0 K 2024-12-09T19:04:32,148 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/00d0787ea1ab40619e8710ae3fc8ce33, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/07f4375a5eae4cf094535d1096a2f709, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2a2fb6687118461a90b91c63bfc335b6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/461173547c544569a815e74dea1d44f4] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=47.0 K 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00d0787ea1ab40619e8710ae3fc8ce33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733771066970 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting d91bcb7329c548a3ac7a6328bb1eeadf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733771066970 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d9df0472b04488a77282192004b6b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771067100 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f4375a5eae4cf094535d1096a2f709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771067100 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 83eb7da0ca0c4340bf24b80a6af1640b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771069001 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2fb6687118461a90b91c63bfc335b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771069001 2024-12-09T19:04:32,149 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b158644f6044f8bf52cacbc58690fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771069726 2024-12-09T19:04:32,150 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461173547c544569a815e74dea1d44f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771069726 2024-12-09T19:04:32,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T19:04:32,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:32,155 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:04:32,158 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:32,159 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/016a90f00d0f42e6ba913d08651a252b is 50, key is test_row_0/A:col10/1733771069726/Put/seqid=0 2024-12-09T19:04:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/01c9ca855f214177a3121831320e311c is 50, key is test_row_0/A:col10/1733771070882/Put/seqid=0 2024-12-09T19:04:32,161 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:32,161 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/90b140e7bb4a4a2a8b38485873910d74 is 50, key is test_row_0/B:col10/1733771069726/Put/seqid=0 2024-12-09T19:04:32,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742054_1230 (size=12001) 2024-12-09T19:04:32,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742055_1231 (size=12241) 2024-12-09T19:04:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742056_1232 (size=12241) 2024-12-09T19:04:32,203 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/90b140e7bb4a4a2a8b38485873910d74 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/90b140e7bb4a4a2a8b38485873910d74 2024-12-09T19:04:32,207 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into 90b140e7bb4a4a2a8b38485873910d74(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:32,207 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:32,207 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=12, startTime=1733771072147; duration=0sec 2024-12-09T19:04:32,207 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:32,207 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:32,207 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:04:32,208 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:04:32,209 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:32,209 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:32,209 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6ff6f1be9a514c50bc856ce60f049c98, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6799a237693e47ec9563703c881a1637, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1d5d259866744d0cab6d13cde0e44a98, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/598d09c3686044f79e7d7d67c483fc83] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=47.0 K 2024-12-09T19:04:32,209 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ff6f1be9a514c50bc856ce60f049c98, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733771066970 2024-12-09T19:04:32,209 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 6799a237693e47ec9563703c881a1637, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771067100 2024-12-09T19:04:32,209 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5d259866744d0cab6d13cde0e44a98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771069001 2024-12-09T19:04:32,210 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 598d09c3686044f79e7d7d67c483fc83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771069726 2024-12-09T19:04:32,216 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:32,216 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/880a567de55141a7b7ae93a58710dd84 is 50, key is test_row_0/C:col10/1733771069726/Put/seqid=0 2024-12-09T19:04:32,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742057_1233 (size=12241) 2024-12-09T19:04:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T19:04:32,225 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/880a567de55141a7b7ae93a58710dd84 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/880a567de55141a7b7ae93a58710dd84 2024-12-09T19:04:32,230 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into 880a567de55141a7b7ae93a58710dd84(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:32,230 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:32,230 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=12, startTime=1733771072147; duration=0sec 2024-12-09T19:04:32,230 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:32,230 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:32,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/01c9ca855f214177a3121831320e311c 2024-12-09T19:04:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/f93bebfe16a94f52b58fe04515279297 is 50, key is test_row_0/B:col10/1733771070882/Put/seqid=0 2024-12-09T19:04:32,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742058_1234 (size=12001) 2024-12-09T19:04:32,598 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/016a90f00d0f42e6ba913d08651a252b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/016a90f00d0f42e6ba913d08651a252b 2024-12-09T19:04:32,602 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 016a90f00d0f42e6ba913d08651a252b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:32,603 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:32,603 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=12, startTime=1733771072147; duration=0sec 2024-12-09T19:04:32,603 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:32,603 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:32,993 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/f93bebfe16a94f52b58fe04515279297 2024-12-09T19:04:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/944dd1b0e32c43a494885b6027c05c2e is 50, key is test_row_0/C:col10/1733771070882/Put/seqid=0 2024-12-09T19:04:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742059_1235 (size=12001) 2024-12-09T19:04:33,010 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/944dd1b0e32c43a494885b6027c05c2e 2024-12-09T19:04:33,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/01c9ca855f214177a3121831320e311c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/01c9ca855f214177a3121831320e311c 2024-12-09T19:04:33,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/01c9ca855f214177a3121831320e311c, entries=150, sequenceid=128, filesize=11.7 K 2024-12-09T19:04:33,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/f93bebfe16a94f52b58fe04515279297 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f93bebfe16a94f52b58fe04515279297 2024-12-09T19:04:33,023 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f93bebfe16a94f52b58fe04515279297, entries=150, sequenceid=128, filesize=11.7 K 2024-12-09T19:04:33,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/944dd1b0e32c43a494885b6027c05c2e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/944dd1b0e32c43a494885b6027c05c2e 2024-12-09T19:04:33,028 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/944dd1b0e32c43a494885b6027c05c2e, entries=150, sequenceid=128, filesize=11.7 K 2024-12-09T19:04:33,029 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for f95442f56e862cbd2c6f1b449c5bb8c9 in 873ms, sequenceid=128, compaction requested=false 2024-12-09T19:04:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T19:04:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T19:04:33,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T19:04:33,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-12-09T19:04:33,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9470 sec 2024-12-09T19:04:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:33,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:04:33,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/1b448b7c10d74d4bb21e39d87a446518 is 50, key is test_row_0/A:col10/1733771073071/Put/seqid=0 2024-12-09T19:04:33,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742060_1236 (size=14541) 2024-12-09T19:04:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771133155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T19:04:33,235 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:33,235 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T19:04:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T19:04:33,237 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:33,237 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:33,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771133264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T19:04:33,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T19:04:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:33,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:33,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771133474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:33,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/1b448b7c10d74d4bb21e39d87a446518 2024-12-09T19:04:33,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/f97892ca7e224af5b593c66624a70576 is 50, key is test_row_0/B:col10/1733771073071/Put/seqid=0 2024-12-09T19:04:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742061_1237 (size=12151) 2024-12-09T19:04:33,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/f97892ca7e224af5b593c66624a70576 2024-12-09T19:04:33,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/de5ea38bc6b34188a68ae9f0d6349403 is 50, key is test_row_0/C:col10/1733771073071/Put/seqid=0 2024-12-09T19:04:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742062_1238 (size=12151) 2024-12-09T19:04:33,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/de5ea38bc6b34188a68ae9f0d6349403 2024-12-09T19:04:33,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/1b448b7c10d74d4bb21e39d87a446518 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/1b448b7c10d74d4bb21e39d87a446518 2024-12-09T19:04:33,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/1b448b7c10d74d4bb21e39d87a446518, entries=200, sequenceid=142, filesize=14.2 K 2024-12-09T19:04:33,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/f97892ca7e224af5b593c66624a70576 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f97892ca7e224af5b593c66624a70576 2024-12-09T19:04:33,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f97892ca7e224af5b593c66624a70576, entries=150, sequenceid=142, filesize=11.9 K 2024-12-09T19:04:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/de5ea38bc6b34188a68ae9f0d6349403 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/de5ea38bc6b34188a68ae9f0d6349403 2024-12-09T19:04:33,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/de5ea38bc6b34188a68ae9f0d6349403, entries=150, sequenceid=142, filesize=11.9 K 2024-12-09T19:04:33,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f95442f56e862cbd2c6f1b449c5bb8c9 in 449ms, sequenceid=142, compaction requested=true 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:33,533 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:33,533 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:33,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:33,534 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:33,534 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:33,534 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,534 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/016a90f00d0f42e6ba913d08651a252b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/01c9ca855f214177a3121831320e311c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/1b448b7c10d74d4bb21e39d87a446518] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=37.9 K 2024-12-09T19:04:33,534 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:33,534 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:33,534 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,535 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/90b140e7bb4a4a2a8b38485873910d74, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f93bebfe16a94f52b58fe04515279297, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f97892ca7e224af5b593c66624a70576] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.5 K 2024-12-09T19:04:33,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 016a90f00d0f42e6ba913d08651a252b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771069726 2024-12-09T19:04:33,535 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b140e7bb4a4a2a8b38485873910d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771069726 2024-12-09T19:04:33,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c9ca855f214177a3121831320e311c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733771070882 2024-12-09T19:04:33,535 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f93bebfe16a94f52b58fe04515279297, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733771070882 2024-12-09T19:04:33,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b448b7c10d74d4bb21e39d87a446518, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733771073071 2024-12-09T19:04:33,535 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f97892ca7e224af5b593c66624a70576, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733771073071 2024-12-09T19:04:33,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T19:04:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,542 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:04:33,545 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:33,545 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:33,545 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/11ef0f34da53417d8a7f812171230adb is 50, key is test_row_0/A:col10/1733771073071/Put/seqid=0 2024-12-09T19:04:33,545 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/0a3c4efa777e486c8ea4212e0acde4e1 is 50, key is test_row_0/B:col10/1733771073071/Put/seqid=0 2024-12-09T19:04:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/4d6f4dbe10ab4ff8af7146d077245cc2 is 50, key is test_row_0/A:col10/1733771073150/Put/seqid=0 2024-12-09T19:04:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T19:04:33,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742064_1240 (size=12493) 2024-12-09T19:04:33,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742063_1239 (size=12493) 2024-12-09T19:04:33,578 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/0a3c4efa777e486c8ea4212e0acde4e1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0a3c4efa777e486c8ea4212e0acde4e1 2024-12-09T19:04:33,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742065_1241 (size=12151) 2024-12-09T19:04:33,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/4d6f4dbe10ab4ff8af7146d077245cc2 2024-12-09T19:04:33,583 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into 0a3c4efa777e486c8ea4212e0acde4e1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:33,583 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:33,583 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771073533; duration=0sec 2024-12-09T19:04:33,583 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:33,583 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:33,583 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:33,584 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:33,584 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:33,584 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:33,584 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/880a567de55141a7b7ae93a58710dd84, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/944dd1b0e32c43a494885b6027c05c2e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/de5ea38bc6b34188a68ae9f0d6349403] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.5 K 2024-12-09T19:04:33,585 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 880a567de55141a7b7ae93a58710dd84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771069726 2024-12-09T19:04:33,585 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 944dd1b0e32c43a494885b6027c05c2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733771070882 2024-12-09T19:04:33,586 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting de5ea38bc6b34188a68ae9f0d6349403, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733771073071 2024-12-09T19:04:33,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/58739913a3ee44c28b800f42bd1fdefa is 50, key is test_row_0/B:col10/1733771073150/Put/seqid=0 2024-12-09T19:04:33,593 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:33,594 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/eec2628a5ab142b1994286e26e70e346 is 50, key is test_row_0/C:col10/1733771073071/Put/seqid=0 2024-12-09T19:04:33,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742067_1243 (size=12493) 2024-12-09T19:04:33,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742066_1242 (size=12151) 2024-12-09T19:04:33,619 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/58739913a3ee44c28b800f42bd1fdefa 2024-12-09T19:04:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/2f88829d7c3b4e27b566a11fbaf51594 is 50, key is test_row_0/C:col10/1733771073150/Put/seqid=0 2024-12-09T19:04:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742068_1244 (size=12151) 2024-12-09T19:04:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:33,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:33,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771133806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T19:04:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771133915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:33,970 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/11ef0f34da53417d8a7f812171230adb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/11ef0f34da53417d8a7f812171230adb 2024-12-09T19:04:33,974 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 11ef0f34da53417d8a7f812171230adb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:33,974 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:33,974 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771073533; duration=0sec 2024-12-09T19:04:33,974 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:33,974 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:34,023 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/eec2628a5ab142b1994286e26e70e346 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eec2628a5ab142b1994286e26e70e346 2024-12-09T19:04:34,027 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into eec2628a5ab142b1994286e26e70e346(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:34,027 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:34,027 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771073533; duration=0sec 2024-12-09T19:04:34,027 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:34,027 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:34,053 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/2f88829d7c3b4e27b566a11fbaf51594 2024-12-09T19:04:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/4d6f4dbe10ab4ff8af7146d077245cc2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4d6f4dbe10ab4ff8af7146d077245cc2 2024-12-09T19:04:34,062 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4d6f4dbe10ab4ff8af7146d077245cc2, entries=150, sequenceid=167, filesize=11.9 K 2024-12-09T19:04:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/58739913a3ee44c28b800f42bd1fdefa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/58739913a3ee44c28b800f42bd1fdefa 2024-12-09T19:04:34,067 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/58739913a3ee44c28b800f42bd1fdefa, entries=150, sequenceid=167, filesize=11.9 K 2024-12-09T19:04:34,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/2f88829d7c3b4e27b566a11fbaf51594 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2f88829d7c3b4e27b566a11fbaf51594 2024-12-09T19:04:34,072 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2f88829d7c3b4e27b566a11fbaf51594, entries=150, sequenceid=167, filesize=11.9 K 2024-12-09T19:04:34,073 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f95442f56e862cbd2c6f1b449c5bb8c9 in 531ms, sequenceid=167, compaction requested=false 2024-12-09T19:04:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T19:04:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T19:04:34,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T19:04:34,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 837 msec 2024-12-09T19:04:34,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 841 msec 2024-12-09T19:04:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:34,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:34,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:04:34,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/6a980a1196f6416995be20c7c09609af is 50, key is test_row_0/A:col10/1733771073805/Put/seqid=0 2024-12-09T19:04:34,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742069_1245 (size=14541) 2024-12-09T19:04:34,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/6a980a1196f6416995be20c7c09609af 2024-12-09T19:04:34,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/ddc665425e544e609e8bd70cb496682c is 50, key is test_row_0/B:col10/1733771073805/Put/seqid=0 2024-12-09T19:04:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742070_1246 (size=12151) 2024-12-09T19:04:34,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771134200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:34,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771134304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T19:04:34,374 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:34,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T19:04:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T19:04:34,377 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:34,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:34,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T19:04:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771134514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:34,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T19:04:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/ddc665425e544e609e8bd70cb496682c 2024-12-09T19:04:34,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/eb5a67eb03984831bd635eca9735194c is 50, key is test_row_0/C:col10/1733771073805/Put/seqid=0 2024-12-09T19:04:34,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742071_1247 (size=12151) 2024-12-09T19:04:34,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T19:04:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T19:04:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771134824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:34,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T19:04:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:34,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/eb5a67eb03984831bd635eca9735194c 2024-12-09T19:04:34,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/6a980a1196f6416995be20c7c09609af as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/6a980a1196f6416995be20c7c09609af 2024-12-09T19:04:34,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/6a980a1196f6416995be20c7c09609af, entries=200, sequenceid=182, filesize=14.2 K 2024-12-09T19:04:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/ddc665425e544e609e8bd70cb496682c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ddc665425e544e609e8bd70cb496682c 2024-12-09T19:04:34,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ddc665425e544e609e8bd70cb496682c, entries=150, sequenceid=182, filesize=11.9 K 2024-12-09T19:04:34,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/eb5a67eb03984831bd635eca9735194c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eb5a67eb03984831bd635eca9735194c 2024-12-09T19:04:34,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eb5a67eb03984831bd635eca9735194c, entries=150, sequenceid=182, filesize=11.9 K 2024-12-09T19:04:34,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f95442f56e862cbd2c6f1b449c5bb8c9 in 858ms, sequenceid=182, compaction requested=true 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:34,984 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:34,984 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:34,985 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:34,985 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:34,985 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:34,985 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:34,985 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,985 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,985 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0a3c4efa777e486c8ea4212e0acde4e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/58739913a3ee44c28b800f42bd1fdefa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ddc665425e544e609e8bd70cb496682c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.9 K 2024-12-09T19:04:34,985 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/11ef0f34da53417d8a7f812171230adb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4d6f4dbe10ab4ff8af7146d077245cc2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/6a980a1196f6416995be20c7c09609af] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=38.3 K 2024-12-09T19:04:34,986 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3c4efa777e486c8ea4212e0acde4e1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733771073071 2024-12-09T19:04:34,986 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ef0f34da53417d8a7f812171230adb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733771073071 2024-12-09T19:04:34,986 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d6f4dbe10ab4ff8af7146d077245cc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733771073141 2024-12-09T19:04:34,986 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 58739913a3ee44c28b800f42bd1fdefa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733771073141 2024-12-09T19:04:34,986 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ddc665425e544e609e8bd70cb496682c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733771073802 2024-12-09T19:04:34,987 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a980a1196f6416995be20c7c09609af, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733771073798 2024-12-09T19:04:34,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T19:04:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:34,988 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:04:34,995 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#208 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/2b3f16f58a3c4a218782694aa994a2c0 is 50, key is test_row_0/A:col10/1733771074166/Put/seqid=0 2024-12-09T19:04:34,995 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/08cb0de2817f4ef084f6095f13ac6054 is 50, key is test_row_0/A:col10/1733771073805/Put/seqid=0 2024-12-09T19:04:34,995 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:34,996 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/9395448e805d4a3fb2e3ddfde804cd34 is 50, key is test_row_0/B:col10/1733771073805/Put/seqid=0 2024-12-09T19:04:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T19:04:35,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742073_1249 (size=12595) 2024-12-09T19:04:35,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742074_1250 (size=12595) 2024-12-09T19:04:35,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742072_1248 (size=12151) 2024-12-09T19:04:35,037 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/9395448e805d4a3fb2e3ddfde804cd34 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/9395448e805d4a3fb2e3ddfde804cd34 2024-12-09T19:04:35,042 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into 9395448e805d4a3fb2e3ddfde804cd34(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:35,042 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:35,042 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771074984; duration=0sec 2024-12-09T19:04:35,042 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:35,042 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:35,042 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:35,043 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:35,043 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:35,043 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:35,044 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eec2628a5ab142b1994286e26e70e346, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2f88829d7c3b4e27b566a11fbaf51594, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eb5a67eb03984831bd635eca9735194c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=35.9 K 2024-12-09T19:04:35,044 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting eec2628a5ab142b1994286e26e70e346, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733771073071 2024-12-09T19:04:35,044 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f88829d7c3b4e27b566a11fbaf51594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733771073141 2024-12-09T19:04:35,044 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting eb5a67eb03984831bd635eca9735194c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733771073802 2024-12-09T19:04:35,051 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:35,052 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/8619e7b0cdf6475fa3a7822c4cee5e1e is 50, key is test_row_0/C:col10/1733771073805/Put/seqid=0 2024-12-09T19:04:35,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742075_1251 (size=12595) 2024-12-09T19:04:35,066 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/8619e7b0cdf6475fa3a7822c4cee5e1e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8619e7b0cdf6475fa3a7822c4cee5e1e 2024-12-09T19:04:35,072 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into 8619e7b0cdf6475fa3a7822c4cee5e1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:35,072 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:35,073 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771074984; duration=0sec 2024-12-09T19:04:35,073 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:35,073 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:35,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:35,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771135309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771135309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56502 deadline: 1733771135314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,317 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771135334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56542 deadline: 1733771135344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,346 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771135424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771135424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,427 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/08cb0de2817f4ef084f6095f13ac6054 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/08cb0de2817f4ef084f6095f13ac6054 2024-12-09T19:04:35,432 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 08cb0de2817f4ef084f6095f13ac6054(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:35,432 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771074984; duration=0sec 2024-12-09T19:04:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:35,434 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/2b3f16f58a3c4a218782694aa994a2c0 2024-12-09T19:04:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/42ad896027b748508f522d5bb9bd3718 is 50, key is test_row_0/B:col10/1733771074166/Put/seqid=0 2024-12-09T19:04:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742076_1252 (size=12151) 2024-12-09T19:04:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T19:04:35,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771135634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771135634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,846 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/42ad896027b748508f522d5bb9bd3718 2024-12-09T19:04:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/3a717dcb49ed45f7adb2e697a377844e is 50, key is test_row_0/C:col10/1733771074166/Put/seqid=0 2024-12-09T19:04:35,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742077_1253 (size=12151) 2024-12-09T19:04:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771135944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771135944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:36,258 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/3a717dcb49ed45f7adb2e697a377844e 2024-12-09T19:04:36,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/2b3f16f58a3c4a218782694aa994a2c0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2b3f16f58a3c4a218782694aa994a2c0 2024-12-09T19:04:36,266 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2b3f16f58a3c4a218782694aa994a2c0, entries=150, sequenceid=206, filesize=11.9 K 2024-12-09T19:04:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/42ad896027b748508f522d5bb9bd3718 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/42ad896027b748508f522d5bb9bd3718 2024-12-09T19:04:36,270 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/42ad896027b748508f522d5bb9bd3718, entries=150, sequenceid=206, filesize=11.9 K 2024-12-09T19:04:36,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/3a717dcb49ed45f7adb2e697a377844e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/3a717dcb49ed45f7adb2e697a377844e 2024-12-09T19:04:36,274 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/3a717dcb49ed45f7adb2e697a377844e, entries=150, sequenceid=206, filesize=11.9 K 2024-12-09T19:04:36,275 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1287ms, sequenceid=206, compaction requested=false 2024-12-09T19:04:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T19:04:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T19:04:36,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T19:04:36,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8990 sec 2024-12-09T19:04:36,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9080 sec 2024-12-09T19:04:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:36,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-09T19:04:36,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/90e585344b194ec59e25133eecc2904f is 50, key is test_row_0/A:col10/1733771076345/Put/seqid=0 2024-12-09T19:04:36,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742078_1254 (size=14541) 2024-12-09T19:04:36,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771136411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:36,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:36,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771136454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:36,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771136454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T19:04:36,524 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:36,524 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T19:04:36,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:36,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771136524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:36,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:36,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:36,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771136735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:36,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/90e585344b194ec59e25133eecc2904f 2024-12-09T19:04:36,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/6f6a69c2db2b4958901ceb4a1df25f5e is 50, key is test_row_0/B:col10/1733771076345/Put/seqid=0 2024-12-09T19:04:36,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742079_1255 (size=12151) 2024-12-09T19:04:36,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:36,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:36,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771137044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:37,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:37,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/6f6a69c2db2b4958901ceb4a1df25f5e 2024-12-09T19:04:37,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1c63624e5ded4a36aeae56d9dae9987c is 50, key is test_row_0/C:col10/1733771076345/Put/seqid=0 2024-12-09T19:04:37,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742080_1256 (size=12151) 2024-12-09T19:04:37,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771137464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:37,466 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:37,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771137474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:37,476 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:37,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771137554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:37,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:37,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1c63624e5ded4a36aeae56d9dae9987c 2024-12-09T19:04:37,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/90e585344b194ec59e25133eecc2904f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/90e585344b194ec59e25133eecc2904f 2024-12-09T19:04:37,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/90e585344b194ec59e25133eecc2904f, entries=200, sequenceid=223, filesize=14.2 K 2024-12-09T19:04:37,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/6f6a69c2db2b4958901ceb4a1df25f5e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/6f6a69c2db2b4958901ceb4a1df25f5e 2024-12-09T19:04:37,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/6f6a69c2db2b4958901ceb4a1df25f5e, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T19:04:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1c63624e5ded4a36aeae56d9dae9987c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1c63624e5ded4a36aeae56d9dae9987c 2024-12-09T19:04:37,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1c63624e5ded4a36aeae56d9dae9987c, entries=150, sequenceid=223, filesize=11.9 K 2024-12-09T19:04:37,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1290ms, sequenceid=223, compaction requested=true 2024-12-09T19:04:37,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:37,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:37,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:37,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:37,636 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:37,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:37,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:37,637 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:37,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:37,638 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:37,638 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:37,638 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,638 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/9395448e805d4a3fb2e3ddfde804cd34, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/42ad896027b748508f522d5bb9bd3718, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/6f6a69c2db2b4958901ceb4a1df25f5e] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.0 K 2024-12-09T19:04:37,638 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:37,638 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:37,638 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,638 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/08cb0de2817f4ef084f6095f13ac6054, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2b3f16f58a3c4a218782694aa994a2c0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/90e585344b194ec59e25133eecc2904f] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=38.4 K 2024-12-09T19:04:37,639 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08cb0de2817f4ef084f6095f13ac6054, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733771073802 2024-12-09T19:04:37,639 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9395448e805d4a3fb2e3ddfde804cd34, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733771073802 2024-12-09T19:04:37,639 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3f16f58a3c4a218782694aa994a2c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733771074166 2024-12-09T19:04:37,639 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 42ad896027b748508f522d5bb9bd3718, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733771074166 2024-12-09T19:04:37,639 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e585344b194ec59e25133eecc2904f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733771075293 2024-12-09T19:04:37,639 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f6a69c2db2b4958901ceb4a1df25f5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733771075293 2024-12-09T19:04:37,647 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:37,647 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/1695b47337fc41d9b2c4cbeb688c1d26 is 50, key is test_row_0/B:col10/1733771076345/Put/seqid=0 2024-12-09T19:04:37,648 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:37,648 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/4f2a149eadff435491afb657a5df8ae0 is 50, key is test_row_0/A:col10/1733771076345/Put/seqid=0 2024-12-09T19:04:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:37,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742082_1258 (size=12697) 2024-12-09T19:04:37,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742081_1257 (size=12697) 2024-12-09T19:04:37,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T19:04:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:37,747 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-09T19:04:37,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/a9555b2a2c2a477fae25403caf92bca8 is 50, key is test_row_0/A:col10/1733771076410/Put/seqid=0 2024-12-09T19:04:37,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742083_1259 (size=12151) 2024-12-09T19:04:38,078 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/1695b47337fc41d9b2c4cbeb688c1d26 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1695b47337fc41d9b2c4cbeb688c1d26 2024-12-09T19:04:38,081 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/4f2a149eadff435491afb657a5df8ae0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4f2a149eadff435491afb657a5df8ae0 2024-12-09T19:04:38,082 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into 1695b47337fc41d9b2c4cbeb688c1d26(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:38,082 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:38,082 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771077636; duration=0sec 2024-12-09T19:04:38,082 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:38,082 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:38,082 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:38,083 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:38,083 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:38,083 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:38,083 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8619e7b0cdf6475fa3a7822c4cee5e1e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/3a717dcb49ed45f7adb2e697a377844e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1c63624e5ded4a36aeae56d9dae9987c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.0 K 2024-12-09T19:04:38,083 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8619e7b0cdf6475fa3a7822c4cee5e1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733771073802 2024-12-09T19:04:38,084 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a717dcb49ed45f7adb2e697a377844e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733771074166 2024-12-09T19:04:38,084 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c63624e5ded4a36aeae56d9dae9987c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733771075293 2024-12-09T19:04:38,086 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 4f2a149eadff435491afb657a5df8ae0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:38,086 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:38,086 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771077636; duration=0sec 2024-12-09T19:04:38,086 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:38,086 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:38,094 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:38,095 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/4ecf1dbda5b94f949dc193fb5b5408c6 is 50, key is test_row_0/C:col10/1733771076345/Put/seqid=0 2024-12-09T19:04:38,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742084_1260 (size=12697) 2024-12-09T19:04:38,103 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/4ecf1dbda5b94f949dc193fb5b5408c6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/4ecf1dbda5b94f949dc193fb5b5408c6 2024-12-09T19:04:38,113 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into 4ecf1dbda5b94f949dc193fb5b5408c6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:38,113 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:38,113 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771077637; duration=0sec 2024-12-09T19:04:38,113 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:38,113 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:38,155 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/a9555b2a2c2a477fae25403caf92bca8 2024-12-09T19:04:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/bd28603c0433492c8e41a9fa7ca59080 is 50, key is test_row_0/B:col10/1733771076410/Put/seqid=0 2024-12-09T19:04:38,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742085_1261 (size=12151) 2024-12-09T19:04:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:38,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:38,585 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/bd28603c0433492c8e41a9fa7ca59080 2024-12-09T19:04:38,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/9e5f3f6a77614ddcb4441922f81e8c35 is 50, key is test_row_0/C:col10/1733771076410/Put/seqid=0 2024-12-09T19:04:38,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742086_1262 (size=12151) 2024-12-09T19:04:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771138606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771138714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771138924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:38,995 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/9e5f3f6a77614ddcb4441922f81e8c35 2024-12-09T19:04:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/a9555b2a2c2a477fae25403caf92bca8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/a9555b2a2c2a477fae25403caf92bca8 2024-12-09T19:04:39,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/a9555b2a2c2a477fae25403caf92bca8, entries=150, sequenceid=245, filesize=11.9 K 2024-12-09T19:04:39,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/bd28603c0433492c8e41a9fa7ca59080 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bd28603c0433492c8e41a9fa7ca59080 2024-12-09T19:04:39,009 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bd28603c0433492c8e41a9fa7ca59080, entries=150, sequenceid=245, filesize=11.9 K 2024-12-09T19:04:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/9e5f3f6a77614ddcb4441922f81e8c35 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/9e5f3f6a77614ddcb4441922f81e8c35 2024-12-09T19:04:39,014 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/9e5f3f6a77614ddcb4441922f81e8c35, entries=150, sequenceid=245, filesize=11.9 K 2024-12-09T19:04:39,015 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1268ms, sequenceid=245, compaction requested=false 2024-12-09T19:04:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:39,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T19:04:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T19:04:39,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T19:04:39,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4900 sec 2024-12-09T19:04:39,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.4950 sec 2024-12-09T19:04:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:39,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:39,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-09T19:04:39,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/47092f25ce3b42eca8025871b102bcbf is 50, key is test_row_0/A:col10/1733771079236/Put/seqid=0 2024-12-09T19:04:39,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742087_1263 (size=14691) 2024-12-09T19:04:39,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771139296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:39,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771139404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:39,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771139485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:39,487 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:39,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771139494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:39,495 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771139614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:39,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/47092f25ce3b42eca8025871b102bcbf 2024-12-09T19:04:39,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/2b91b7ddf8564927baf075f2ae13c7a8 is 50, key is test_row_0/B:col10/1733771079236/Put/seqid=0 2024-12-09T19:04:39,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742088_1264 (size=12301) 2024-12-09T19:04:39,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771139924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:40,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/2b91b7ddf8564927baf075f2ae13c7a8 2024-12-09T19:04:40,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e is 50, key is test_row_0/C:col10/1733771079236/Put/seqid=0 2024-12-09T19:04:40,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742089_1265 (size=12301) 2024-12-09T19:04:40,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771140434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:40,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e 2024-12-09T19:04:40,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/47092f25ce3b42eca8025871b102bcbf as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/47092f25ce3b42eca8025871b102bcbf 2024-12-09T19:04:40,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/47092f25ce3b42eca8025871b102bcbf, entries=200, sequenceid=263, filesize=14.3 K 2024-12-09T19:04:40,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/2b91b7ddf8564927baf075f2ae13c7a8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/2b91b7ddf8564927baf075f2ae13c7a8 2024-12-09T19:04:40,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/2b91b7ddf8564927baf075f2ae13c7a8, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T19:04:40,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e 2024-12-09T19:04:40,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e, entries=150, sequenceid=263, filesize=12.0 K 2024-12-09T19:04:40,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1268ms, sequenceid=263, compaction requested=true 2024-12-09T19:04:40,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:40,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:40,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:40,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:40,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:40,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:40,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:40,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:40,507 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:40,507 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:40,507 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1695b47337fc41d9b2c4cbeb688c1d26, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bd28603c0433492c8e41a9fa7ca59080, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/2b91b7ddf8564927baf075f2ae13c7a8] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.3 K 2024-12-09T19:04:40,507 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4f2a149eadff435491afb657a5df8ae0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/a9555b2a2c2a477fae25403caf92bca8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/47092f25ce3b42eca8025871b102bcbf] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=38.6 K 2024-12-09T19:04:40,507 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f2a149eadff435491afb657a5df8ae0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733771075293 2024-12-09T19:04:40,507 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1695b47337fc41d9b2c4cbeb688c1d26, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733771075293 2024-12-09T19:04:40,507 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9555b2a2c2a477fae25403caf92bca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733771076394 2024-12-09T19:04:40,507 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting bd28603c0433492c8e41a9fa7ca59080, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733771076394 2024-12-09T19:04:40,508 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47092f25ce3b42eca8025871b102bcbf, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733771078584 2024-12-09T19:04:40,508 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b91b7ddf8564927baf075f2ae13c7a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733771078599 2024-12-09T19:04:40,514 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:40,514 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/5ee3240ed3aa43b9930b72bcedf74ac6 is 50, key is test_row_0/A:col10/1733771079236/Put/seqid=0 2024-12-09T19:04:40,529 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:40,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742090_1266 (size=12949) 2024-12-09T19:04:40,530 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/ce61a956c5084d6f96e0841a646b0a10 is 50, key is test_row_0/B:col10/1733771079236/Put/seqid=0 2024-12-09T19:04:40,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742091_1267 (size=12949) 2024-12-09T19:04:40,538 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/ce61a956c5084d6f96e0841a646b0a10 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ce61a956c5084d6f96e0841a646b0a10 2024-12-09T19:04:40,542 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into ce61a956c5084d6f96e0841a646b0a10(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:40,542 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:40,542 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771080506; duration=0sec 2024-12-09T19:04:40,542 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:40,542 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:40,542 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:40,543 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:40,543 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:40,543 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:40,543 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/4ecf1dbda5b94f949dc193fb5b5408c6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/9e5f3f6a77614ddcb4441922f81e8c35, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.3 K 2024-12-09T19:04:40,543 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecf1dbda5b94f949dc193fb5b5408c6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733771075293 2024-12-09T19:04:40,543 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e5f3f6a77614ddcb4441922f81e8c35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733771076394 2024-12-09T19:04:40,544 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f78b9f1e5c1640c5a2e1d3b0060a5b1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733771078599 2024-12-09T19:04:40,549 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:40,550 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/a1f26cead2bd4b27bd4a9b15dca97546 is 50, key is test_row_0/C:col10/1733771079236/Put/seqid=0 2024-12-09T19:04:40,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742092_1268 (size=12949) 2024-12-09T19:04:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T19:04:40,694 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:40,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T19:04:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T19:04:40,696 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:40,697 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:40,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T19:04:40,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T19:04:40,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:40,849 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-09T19:04:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/d7dcec2e26ac4264b9e5a528a844e21b is 50, key is test_row_0/A:col10/1733771079265/Put/seqid=0 2024-12-09T19:04:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742093_1269 (size=12301) 2024-12-09T19:04:40,866 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/d7dcec2e26ac4264b9e5a528a844e21b 2024-12-09T19:04:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/4727c01cbecc4240b66756747b86c822 is 50, key is test_row_0/B:col10/1733771079265/Put/seqid=0 2024-12-09T19:04:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742094_1270 (size=12301) 2024-12-09T19:04:40,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/5ee3240ed3aa43b9930b72bcedf74ac6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/5ee3240ed3aa43b9930b72bcedf74ac6 2024-12-09T19:04:40,938 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 5ee3240ed3aa43b9930b72bcedf74ac6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:40,938 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:40,938 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771080505; duration=0sec 2024-12-09T19:04:40,938 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:40,938 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:40,959 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/a1f26cead2bd4b27bd4a9b15dca97546 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/a1f26cead2bd4b27bd4a9b15dca97546 2024-12-09T19:04:40,964 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into a1f26cead2bd4b27bd4a9b15dca97546(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:40,964 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:40,964 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771080506; duration=0sec 2024-12-09T19:04:40,964 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:40,964 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T19:04:41,276 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/4727c01cbecc4240b66756747b86c822 2024-12-09T19:04:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/adced595fbaa4da0a174cbe2a8c7ef1d is 50, key is test_row_0/C:col10/1733771079265/Put/seqid=0 2024-12-09T19:04:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742095_1271 (size=12301) 2024-12-09T19:04:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T19:04:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:41,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:41,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771141477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:41,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771141585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:41,689 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/adced595fbaa4da0a174cbe2a8c7ef1d 2024-12-09T19:04:41,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/d7dcec2e26ac4264b9e5a528a844e21b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d7dcec2e26ac4264b9e5a528a844e21b 2024-12-09T19:04:41,697 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d7dcec2e26ac4264b9e5a528a844e21b, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T19:04:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/4727c01cbecc4240b66756747b86c822 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4727c01cbecc4240b66756747b86c822 2024-12-09T19:04:41,700 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4727c01cbecc4240b66756747b86c822, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T19:04:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/adced595fbaa4da0a174cbe2a8c7ef1d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/adced595fbaa4da0a174cbe2a8c7ef1d 2024-12-09T19:04:41,705 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/adced595fbaa4da0a174cbe2a8c7ef1d, entries=150, sequenceid=285, filesize=12.0 K 2024-12-09T19:04:41,705 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for f95442f56e862cbd2c6f1b449c5bb8c9 in 856ms, sequenceid=285, compaction requested=false 2024-12-09T19:04:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T19:04:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T19:04:41,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T19:04:41,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-12-09T19:04:41,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0140 sec 2024-12-09T19:04:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:41,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-09T19:04:41,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/d5c803e07a314e019a97d4c560a02a2e is 50, key is test_row_0/A:col10/1733771081795/Put/seqid=0 2024-12-09T19:04:41,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742096_1272 (size=14741) 2024-12-09T19:04:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T19:04:41,835 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:41,835 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T19:04:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:41,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:41,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:41,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:41,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771141850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:41,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771141954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:41,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T19:04:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:41,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T19:04:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771142165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:42,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/d5c803e07a314e019a97d4c560a02a2e 2024-12-09T19:04:42,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/779a4516dd5c479a80a559f6f1dd64db is 50, key is test_row_0/B:col10/1733771081795/Put/seqid=0 2024-12-09T19:04:42,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742097_1273 (size=12301) 2024-12-09T19:04:42,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T19:04:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T19:04:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771142474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:42,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T19:04:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:42,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/779a4516dd5c479a80a559f6f1dd64db 2024-12-09T19:04:42,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/39fff0b47a254b0fbe7e33fe2e6a2ad9 is 50, key is test_row_0/C:col10/1733771081795/Put/seqid=0 2024-12-09T19:04:42,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742098_1274 (size=12301) 2024-12-09T19:04:42,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/39fff0b47a254b0fbe7e33fe2e6a2ad9 2024-12-09T19:04:42,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/d5c803e07a314e019a97d4c560a02a2e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d5c803e07a314e019a97d4c560a02a2e 2024-12-09T19:04:42,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d5c803e07a314e019a97d4c560a02a2e, entries=200, sequenceid=303, filesize=14.4 K 2024-12-09T19:04:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/779a4516dd5c479a80a559f6f1dd64db as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/779a4516dd5c479a80a559f6f1dd64db 2024-12-09T19:04:42,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/779a4516dd5c479a80a559f6f1dd64db, entries=150, sequenceid=303, filesize=12.0 K 2024-12-09T19:04:42,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/39fff0b47a254b0fbe7e33fe2e6a2ad9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/39fff0b47a254b0fbe7e33fe2e6a2ad9 2024-12-09T19:04:42,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/39fff0b47a254b0fbe7e33fe2e6a2ad9, entries=150, sequenceid=303, filesize=12.0 K 2024-12-09T19:04:42,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f95442f56e862cbd2c6f1b449c5bb8c9 in 862ms, sequenceid=303, compaction requested=true 2024-12-09T19:04:42,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:42,658 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:42,659 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:42,660 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:42,660 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:42,660 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:42,660 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:42,660 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,660 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,660 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ce61a956c5084d6f96e0841a646b0a10, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4727c01cbecc4240b66756747b86c822, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/779a4516dd5c479a80a559f6f1dd64db] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.7 K 2024-12-09T19:04:42,660 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/5ee3240ed3aa43b9930b72bcedf74ac6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d7dcec2e26ac4264b9e5a528a844e21b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d5c803e07a314e019a97d4c560a02a2e] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=39.1 K 2024-12-09T19:04:42,661 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ce61a956c5084d6f96e0841a646b0a10, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733771078599 2024-12-09T19:04:42,661 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee3240ed3aa43b9930b72bcedf74ac6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733771078599 2024-12-09T19:04:42,661 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 4727c01cbecc4240b66756747b86c822, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733771079265 2024-12-09T19:04:42,661 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7dcec2e26ac4264b9e5a528a844e21b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733771079265 2024-12-09T19:04:42,661 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5c803e07a314e019a97d4c560a02a2e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733771081470 2024-12-09T19:04:42,661 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 779a4516dd5c479a80a559f6f1dd64db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733771081473 2024-12-09T19:04:42,668 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:42,668 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:42,668 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/0e7f28e39e5643cd919ce143d6579fad is 50, key is test_row_0/B:col10/1733771081795/Put/seqid=0 2024-12-09T19:04:42,669 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/3e250fb2de82486a87f8078556dad9a5 is 50, key is test_row_0/A:col10/1733771081795/Put/seqid=0 2024-12-09T19:04:42,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742100_1276 (size=13051) 2024-12-09T19:04:42,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742099_1275 (size=13051) 2024-12-09T19:04:42,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T19:04:42,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:42,754 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-09T19:04:42,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/c49a3becdd6f459d9feae544b8fc2e2b is 50, key is test_row_0/A:col10/1733771081848/Put/seqid=0 2024-12-09T19:04:42,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742101_1277 (size=12301) 2024-12-09T19:04:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:42,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. as already flushing 2024-12-09T19:04:43,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771143026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:43,097 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/3e250fb2de82486a87f8078556dad9a5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/3e250fb2de82486a87f8078556dad9a5 2024-12-09T19:04:43,101 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into 3e250fb2de82486a87f8078556dad9a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:43,101 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:43,101 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771082658; duration=0sec 2024-12-09T19:04:43,102 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:43,102 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:43,102 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:43,103 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:43,103 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:43,103 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:43,103 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/a1f26cead2bd4b27bd4a9b15dca97546, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/adced595fbaa4da0a174cbe2a8c7ef1d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/39fff0b47a254b0fbe7e33fe2e6a2ad9] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.7 K 2024-12-09T19:04:43,104 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f26cead2bd4b27bd4a9b15dca97546, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733771078599 2024-12-09T19:04:43,104 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting adced595fbaa4da0a174cbe2a8c7ef1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733771079265 2024-12-09T19:04:43,104 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39fff0b47a254b0fbe7e33fe2e6a2ad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733771081473 2024-12-09T19:04:43,106 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/0e7f28e39e5643cd919ce143d6579fad as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0e7f28e39e5643cd919ce143d6579fad 2024-12-09T19:04:43,111 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:43,111 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into 0e7f28e39e5643cd919ce143d6579fad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:43,111 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:43,111 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771082659; duration=0sec 2024-12-09T19:04:43,111 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:43,111 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:43,111 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/ecc4018bbe204672baa5f9ad4a2ac7d1 is 50, key is test_row_0/C:col10/1733771081795/Put/seqid=0 2024-12-09T19:04:43,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742102_1278 (size=13051) 2024-12-09T19:04:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771143134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:43,165 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/c49a3becdd6f459d9feae544b8fc2e2b 2024-12-09T19:04:43,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/b6a1da9220144417bf4a9ab101f7c9e1 is 50, key is test_row_0/B:col10/1733771081848/Put/seqid=0 2024-12-09T19:04:43,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742103_1279 (size=12301) 2024-12-09T19:04:43,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771143344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56558 deadline: 1733771143514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:43,517 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:43,519 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/ecc4018bbe204672baa5f9ad4a2ac7d1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/ecc4018bbe204672baa5f9ad4a2ac7d1 2024-12-09T19:04:43,523 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into ecc4018bbe204672baa5f9ad4a2ac7d1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:43,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:43,523 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771082659; duration=0sec 2024-12-09T19:04:43,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:43,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56470 deadline: 1733771143525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:43,527 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in f95442f56e862cbd2c6f1b449c5bb8c9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:04:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:43,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/b6a1da9220144417bf4a9ab101f7c9e1 2024-12-09T19:04:43,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/97a9a72a47654402ba90fdc82bac28dd is 50, key is test_row_0/C:col10/1733771081848/Put/seqid=0 2024-12-09T19:04:43,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742104_1280 (size=12301) 2024-12-09T19:04:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771143655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:43,987 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/97a9a72a47654402ba90fdc82bac28dd 2024-12-09T19:04:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/c49a3becdd6f459d9feae544b8fc2e2b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/c49a3becdd6f459d9feae544b8fc2e2b 2024-12-09T19:04:43,995 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/c49a3becdd6f459d9feae544b8fc2e2b, entries=150, sequenceid=323, filesize=12.0 K 2024-12-09T19:04:43,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/b6a1da9220144417bf4a9ab101f7c9e1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/b6a1da9220144417bf4a9ab101f7c9e1 2024-12-09T19:04:43,999 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/b6a1da9220144417bf4a9ab101f7c9e1, entries=150, sequenceid=323, filesize=12.0 K 2024-12-09T19:04:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/97a9a72a47654402ba90fdc82bac28dd as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/97a9a72a47654402ba90fdc82bac28dd 2024-12-09T19:04:44,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/97a9a72a47654402ba90fdc82bac28dd, entries=150, sequenceid=323, filesize=12.0 K 2024-12-09T19:04:44,005 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1250ms, sequenceid=323, compaction requested=false 2024-12-09T19:04:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T19:04:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T19:04:44,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T19:04:44,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-12-09T19:04:44,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1730 sec 2024-12-09T19:04:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:44,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-09T19:04:44,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/aeb98a30c5da4ec883380ab19a533b03 is 50, key is test_row_0/A:col10/1733771083025/Put/seqid=0 2024-12-09T19:04:44,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742105_1281 (size=14741) 2024-12-09T19:04:44,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/aeb98a30c5da4ec883380ab19a533b03 2024-12-09T19:04:44,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/e4a9f67773eb4cf7ac9dd678aebdc8a8 is 50, key is test_row_0/B:col10/1733771083025/Put/seqid=0 2024-12-09T19:04:44,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742106_1282 (size=12301) 2024-12-09T19:04:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771144227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771144335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56526 deadline: 1733771144544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:04:44,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/e4a9f67773eb4cf7ac9dd678aebdc8a8 2024-12-09T19:04:44,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1e4ddb3c0f554eeeb486275371902b28 is 50, key is test_row_0/C:col10/1733771083025/Put/seqid=0 2024-12-09T19:04:44,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742107_1283 (size=12301) 2024-12-09T19:04:44,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1e4ddb3c0f554eeeb486275371902b28 2024-12-09T19:04:44,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/aeb98a30c5da4ec883380ab19a533b03 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/aeb98a30c5da4ec883380ab19a533b03 2024-12-09T19:04:44,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/aeb98a30c5da4ec883380ab19a533b03, entries=200, sequenceid=343, filesize=14.4 K 2024-12-09T19:04:44,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/e4a9f67773eb4cf7ac9dd678aebdc8a8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4a9f67773eb4cf7ac9dd678aebdc8a8 2024-12-09T19:04:44,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4a9f67773eb4cf7ac9dd678aebdc8a8, entries=150, sequenceid=343, filesize=12.0 K 2024-12-09T19:04:44,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/1e4ddb3c0f554eeeb486275371902b28 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1e4ddb3c0f554eeeb486275371902b28 2024-12-09T19:04:44,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1e4ddb3c0f554eeeb486275371902b28, entries=150, sequenceid=343, filesize=12.0 K 2024-12-09T19:04:44,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f95442f56e862cbd2c6f1b449c5bb8c9 in 489ms, sequenceid=343, compaction requested=true 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:44,655 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:44,655 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f95442f56e862cbd2c6f1b449c5bb8c9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:04:44,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:44,656 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:44,656 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/B is initiating minor compaction (all files) 2024-12-09T19:04:44,656 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/B in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:44,656 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0e7f28e39e5643cd919ce143d6579fad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/b6a1da9220144417bf4a9ab101f7c9e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4a9f67773eb4cf7ac9dd678aebdc8a8] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.8 K 2024-12-09T19:04:44,656 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:44,657 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/A is initiating minor compaction (all files) 2024-12-09T19:04:44,657 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/A in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:44,657 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/3e250fb2de82486a87f8078556dad9a5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/c49a3becdd6f459d9feae544b8fc2e2b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/aeb98a30c5da4ec883380ab19a533b03] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=39.2 K 2024-12-09T19:04:44,657 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7f28e39e5643cd919ce143d6579fad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733771081473 2024-12-09T19:04:44,657 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e250fb2de82486a87f8078556dad9a5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733771081473 2024-12-09T19:04:44,657 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a1da9220144417bf4a9ab101f7c9e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771081838 2024-12-09T19:04:44,657 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49a3becdd6f459d9feae544b8fc2e2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771081838 2024-12-09T19:04:44,657 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a9f67773eb4cf7ac9dd678aebdc8a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733771083021 2024-12-09T19:04:44,658 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb98a30c5da4ec883380ab19a533b03, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733771083017 2024-12-09T19:04:44,664 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:44,664 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/dd975a840f3e44a9a54e6bd437165adc is 50, key is test_row_0/A:col10/1733771083025/Put/seqid=0 2024-12-09T19:04:44,670 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:44,671 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/4ec06b3626664fc4a4fe829e29e9f342 is 50, key is test_row_0/B:col10/1733771083025/Put/seqid=0 2024-12-09T19:04:44,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742108_1284 (size=13153) 2024-12-09T19:04:44,682 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/dd975a840f3e44a9a54e6bd437165adc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/dd975a840f3e44a9a54e6bd437165adc 2024-12-09T19:04:44,687 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/A of f95442f56e862cbd2c6f1b449c5bb8c9 into dd975a840f3e44a9a54e6bd437165adc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:44,687 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:44,687 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/A, priority=13, startTime=1733771084655; duration=0sec 2024-12-09T19:04:44,687 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:04:44,687 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:A 2024-12-09T19:04:44,687 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:04:44,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742109_1285 (size=13153) 2024-12-09T19:04:44,689 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:04:44,689 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): f95442f56e862cbd2c6f1b449c5bb8c9/C is initiating minor compaction (all files) 2024-12-09T19:04:44,689 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f95442f56e862cbd2c6f1b449c5bb8c9/C in TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:44,689 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/ecc4018bbe204672baa5f9ad4a2ac7d1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/97a9a72a47654402ba90fdc82bac28dd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1e4ddb3c0f554eeeb486275371902b28] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp, totalSize=36.8 K 2024-12-09T19:04:44,689 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc4018bbe204672baa5f9ad4a2ac7d1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733771081473 2024-12-09T19:04:44,690 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a9a72a47654402ba90fdc82bac28dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771081838 2024-12-09T19:04:44,690 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e4ddb3c0f554eeeb486275371902b28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733771083021 2024-12-09T19:04:44,694 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/4ec06b3626664fc4a4fe829e29e9f342 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4ec06b3626664fc4a4fe829e29e9f342 2024-12-09T19:04:44,700 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/B of f95442f56e862cbd2c6f1b449c5bb8c9 into 4ec06b3626664fc4a4fe829e29e9f342(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:44,701 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:44,701 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/B, priority=13, startTime=1733771084655; duration=0sec 2024-12-09T19:04:44,701 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:44,701 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:B 2024-12-09T19:04:44,702 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f95442f56e862cbd2c6f1b449c5bb8c9#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:04:44,703 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/27e8bc52538d4204a3cd43e910703ac9 is 50, key is test_row_0/C:col10/1733771083025/Put/seqid=0 2024-12-09T19:04:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742110_1286 (size=13153) 2024-12-09T19:04:44,712 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/27e8bc52538d4204a3cd43e910703ac9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/27e8bc52538d4204a3cd43e910703ac9 2024-12-09T19:04:44,718 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f95442f56e862cbd2c6f1b449c5bb8c9/C of f95442f56e862cbd2c6f1b449c5bb8c9 into 27e8bc52538d4204a3cd43e910703ac9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:04:44,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:44,718 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., storeName=f95442f56e862cbd2c6f1b449c5bb8c9/C, priority=13, startTime=1733771084655; duration=0sec 2024-12-09T19:04:44,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:04:44,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f95442f56e862cbd2c6f1b449c5bb8c9:C 2024-12-09T19:04:44,764 INFO [Thread-1011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1011. 2024-12-09T19:04:44,764 INFO [Thread-1007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1007. 2024-12-09T19:04:44,764 DEBUG [Thread-1011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:44,764 DEBUG [Thread-1007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:44,764 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,764 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,764 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,764 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:44,765 INFO [Thread-1005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1005. 2024-12-09T19:04:44,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:44,765 DEBUG [Thread-1005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:44,765 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:44,765 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,766 INFO [Thread-1009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1009. 2024-12-09T19:04:44,766 DEBUG [Thread-1009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:44,766 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,766 INFO [Thread-1003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1003. 2024-12-09T19:04:44,766 DEBUG [Thread-1009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:44,766 DEBUG [Thread-1003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:44,766 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,766 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:44,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-09T19:04:44,859 INFO [Thread-1000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1000. 2024-12-09T19:04:44,860 DEBUG [Thread-1000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:44,860 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,860 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:44,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:44,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/e913528b9cb84cd1b45b049d04c49250 is 50, key is test_row_0/A:col10/1733771084856/Put/seqid=0 2024-12-09T19:04:44,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742111_1287 (size=12301) 2024-12-09T19:04:45,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/e913528b9cb84cd1b45b049d04c49250 2024-12-09T19:04:45,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/bfecffe672c34ac6823ec2f8071beff8 is 50, key is test_row_0/B:col10/1733771084856/Put/seqid=0 2024-12-09T19:04:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742112_1288 (size=12301) 2024-12-09T19:04:45,398 INFO [Thread-994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-994. 2024-12-09T19:04:45,398 DEBUG [Thread-994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:45,398 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:45,398 DEBUG [Thread-994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:45,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:45,408 INFO [Thread-998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-998. 2024-12-09T19:04:45,408 DEBUG [Thread-998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:45,408 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:45,408 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:45,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:45,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/bfecffe672c34ac6823ec2f8071beff8 2024-12-09T19:04:45,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/2c2034ba970844a9b819554fe93a5d2e is 50, key is test_row_0/C:col10/1733771084856/Put/seqid=0 2024-12-09T19:04:45,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742113_1289 (size=12301) 2024-12-09T19:04:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T19:04:46,007 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:46,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/2c2034ba970844a9b819554fe93a5d2e 2024-12-09T19:04:46,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/e913528b9cb84cd1b45b049d04c49250 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/e913528b9cb84cd1b45b049d04c49250 2024-12-09T19:04:46,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/e913528b9cb84cd1b45b049d04c49250, entries=150, sequenceid=366, filesize=12.0 K 2024-12-09T19:04:46,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/bfecffe672c34ac6823ec2f8071beff8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bfecffe672c34ac6823ec2f8071beff8 2024-12-09T19:04:46,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bfecffe672c34ac6823ec2f8071beff8, entries=150, sequenceid=366, filesize=12.0 K 2024-12-09T19:04:46,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/2c2034ba970844a9b819554fe93a5d2e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2c2034ba970844a9b819554fe93a5d2e 2024-12-09T19:04:46,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2c2034ba970844a9b819554fe93a5d2e, entries=150, sequenceid=366, filesize=12.0 K 2024-12-09T19:04:46,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1296ms, sequenceid=366, compaction requested=false 2024-12-09T19:04:46,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f95442f56e862cbd2c6f1b449c5bb8c9: 2024-12-09T19:04:46,699 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T19:04:53,569 INFO [Thread-996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-996. 2024-12-09T19:04:53,569 DEBUG [Thread-996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:53,569 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:53,570 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:53,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:53,596 INFO [Thread-992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-992. 2024-12-09T19:04:53,596 DEBUG [Thread-992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:04:53,596 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:53,596 DEBUG [Thread-992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:53,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 217 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5595 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5265 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5248 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5582 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5266 2024-12-09T19:04:53,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T19:04:53,596 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:04:53,596 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:04:53,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:53,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:53,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:04:53,597 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T19:04:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T19:04:53,600 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771093600"}]},"ts":"1733771093600"} 2024-12-09T19:04:53,601 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T19:04:53,601 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T19:04:53,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:04:53,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, UNASSIGN}] 2024-12-09T19:04:53,604 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, UNASSIGN 2024-12-09T19:04:53,605 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=f95442f56e862cbd2c6f1b449c5bb8c9, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:53,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, UNASSIGN because future has completed 2024-12-09T19:04:53,607 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:04:53,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:04:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T19:04:53,761 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:53,761 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:04:53,761 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing f95442f56e862cbd2c6f1b449c5bb8c9, disabling compactions & flushes 2024-12-09T19:04:53,761 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:53,761 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:53,761 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. after waiting 0 ms 2024-12-09T19:04:53,761 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:53,762 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing f95442f56e862cbd2c6f1b449c5bb8c9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-09T19:04:53,772 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/816336e1aff648478e298e504f2f090b is 50, key is test_row_0/A:col10/1733771085405/Put/seqid=0 2024-12-09T19:04:53,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742114_1290 (size=12301) 2024-12-09T19:04:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T19:04:54,178 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/816336e1aff648478e298e504f2f090b 2024-12-09T19:04:54,191 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/c2f693b3445e4d9294c388beb27efa2a is 50, key is test_row_0/B:col10/1733771085405/Put/seqid=0 2024-12-09T19:04:54,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742115_1291 (size=12301) 2024-12-09T19:04:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T19:04:54,596 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/c2f693b3445e4d9294c388beb27efa2a 2024-12-09T19:04:54,608 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/f14c72c3b2d04dedaa607fe154245b23 is 50, key is test_row_0/C:col10/1733771085405/Put/seqid=0 2024-12-09T19:04:54,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742116_1292 (size=12301) 2024-12-09T19:04:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T19:04:55,014 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/f14c72c3b2d04dedaa607fe154245b23 2024-12-09T19:04:55,023 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/A/816336e1aff648478e298e504f2f090b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/816336e1aff648478e298e504f2f090b 2024-12-09T19:04:55,029 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/816336e1aff648478e298e504f2f090b, entries=150, sequenceid=373, filesize=12.0 K 2024-12-09T19:04:55,030 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/B/c2f693b3445e4d9294c388beb27efa2a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/c2f693b3445e4d9294c388beb27efa2a 2024-12-09T19:04:55,035 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/c2f693b3445e4d9294c388beb27efa2a, entries=150, sequenceid=373, filesize=12.0 K 2024-12-09T19:04:55,037 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/.tmp/C/f14c72c3b2d04dedaa607fe154245b23 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f14c72c3b2d04dedaa607fe154245b23 2024-12-09T19:04:55,041 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f14c72c3b2d04dedaa607fe154245b23, entries=150, sequenceid=373, filesize=12.0 K 2024-12-09T19:04:55,041 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1280ms, sequenceid=373, compaction requested=true 2024-12-09T19:04:55,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/af94ab8912ed420fa7781d6856385f40, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/81cd170175b241a5a381e41bf82b8681, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/00d0787ea1ab40619e8710ae3fc8ce33, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/32e76503a4f6440eb8e070f29b3c4102, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/07f4375a5eae4cf094535d1096a2f709, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2a2fb6687118461a90b91c63bfc335b6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/016a90f00d0f42e6ba913d08651a252b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/461173547c544569a815e74dea1d44f4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/01c9ca855f214177a3121831320e311c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/1b448b7c10d74d4bb21e39d87a446518, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/11ef0f34da53417d8a7f812171230adb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4d6f4dbe10ab4ff8af7146d077245cc2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/6a980a1196f6416995be20c7c09609af, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/08cb0de2817f4ef084f6095f13ac6054, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2b3f16f58a3c4a218782694aa994a2c0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/90e585344b194ec59e25133eecc2904f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4f2a149eadff435491afb657a5df8ae0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/a9555b2a2c2a477fae25403caf92bca8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/47092f25ce3b42eca8025871b102bcbf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/5ee3240ed3aa43b9930b72bcedf74ac6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d7dcec2e26ac4264b9e5a528a844e21b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d5c803e07a314e019a97d4c560a02a2e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/3e250fb2de82486a87f8078556dad9a5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/c49a3becdd6f459d9feae544b8fc2e2b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/aeb98a30c5da4ec883380ab19a533b03] to archive 2024-12-09T19:04:55,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:04:55,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/af94ab8912ed420fa7781d6856385f40 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/af94ab8912ed420fa7781d6856385f40 2024-12-09T19:04:55,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/81cd170175b241a5a381e41bf82b8681 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/81cd170175b241a5a381e41bf82b8681 2024-12-09T19:04:55,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/00d0787ea1ab40619e8710ae3fc8ce33 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/00d0787ea1ab40619e8710ae3fc8ce33 2024-12-09T19:04:55,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/32e76503a4f6440eb8e070f29b3c4102 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/32e76503a4f6440eb8e070f29b3c4102 2024-12-09T19:04:55,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/07f4375a5eae4cf094535d1096a2f709 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/07f4375a5eae4cf094535d1096a2f709 2024-12-09T19:04:55,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2a2fb6687118461a90b91c63bfc335b6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2a2fb6687118461a90b91c63bfc335b6 2024-12-09T19:04:55,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/016a90f00d0f42e6ba913d08651a252b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/016a90f00d0f42e6ba913d08651a252b 2024-12-09T19:04:55,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/461173547c544569a815e74dea1d44f4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/461173547c544569a815e74dea1d44f4 2024-12-09T19:04:55,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/01c9ca855f214177a3121831320e311c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/01c9ca855f214177a3121831320e311c 2024-12-09T19:04:55,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/1b448b7c10d74d4bb21e39d87a446518 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/1b448b7c10d74d4bb21e39d87a446518 2024-12-09T19:04:55,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/11ef0f34da53417d8a7f812171230adb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/11ef0f34da53417d8a7f812171230adb 2024-12-09T19:04:55,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4d6f4dbe10ab4ff8af7146d077245cc2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4d6f4dbe10ab4ff8af7146d077245cc2 2024-12-09T19:04:55,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/6a980a1196f6416995be20c7c09609af to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/6a980a1196f6416995be20c7c09609af 2024-12-09T19:04:55,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/08cb0de2817f4ef084f6095f13ac6054 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/08cb0de2817f4ef084f6095f13ac6054 2024-12-09T19:04:55,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2b3f16f58a3c4a218782694aa994a2c0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/2b3f16f58a3c4a218782694aa994a2c0 2024-12-09T19:04:55,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/90e585344b194ec59e25133eecc2904f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/90e585344b194ec59e25133eecc2904f 2024-12-09T19:04:55,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4f2a149eadff435491afb657a5df8ae0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/4f2a149eadff435491afb657a5df8ae0 2024-12-09T19:04:55,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/a9555b2a2c2a477fae25403caf92bca8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/a9555b2a2c2a477fae25403caf92bca8 2024-12-09T19:04:55,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/47092f25ce3b42eca8025871b102bcbf to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/47092f25ce3b42eca8025871b102bcbf 2024-12-09T19:04:55,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/5ee3240ed3aa43b9930b72bcedf74ac6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/5ee3240ed3aa43b9930b72bcedf74ac6 2024-12-09T19:04:55,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d7dcec2e26ac4264b9e5a528a844e21b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d7dcec2e26ac4264b9e5a528a844e21b 2024-12-09T19:04:55,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d5c803e07a314e019a97d4c560a02a2e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/d5c803e07a314e019a97d4c560a02a2e 2024-12-09T19:04:55,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/3e250fb2de82486a87f8078556dad9a5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/3e250fb2de82486a87f8078556dad9a5 2024-12-09T19:04:55,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/c49a3becdd6f459d9feae544b8fc2e2b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/c49a3becdd6f459d9feae544b8fc2e2b 2024-12-09T19:04:55,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/aeb98a30c5da4ec883380ab19a533b03 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/aeb98a30c5da4ec883380ab19a533b03 2024-12-09T19:04:55,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/fa7465c7a5da490c9f4c9a44b36ba400, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1686c3da175b4dd1840b01278d5c1e1e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/d91bcb7329c548a3ac7a6328bb1eeadf, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4ac258c6e0446cfb78217ee12d9f316, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/76d9df0472b04488a77282192004b6b2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/83eb7da0ca0c4340bf24b80a6af1640b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/90b140e7bb4a4a2a8b38485873910d74, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/22b158644f6044f8bf52cacbc58690fb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f93bebfe16a94f52b58fe04515279297, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0a3c4efa777e486c8ea4212e0acde4e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f97892ca7e224af5b593c66624a70576, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/58739913a3ee44c28b800f42bd1fdefa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/9395448e805d4a3fb2e3ddfde804cd34, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ddc665425e544e609e8bd70cb496682c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/42ad896027b748508f522d5bb9bd3718, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1695b47337fc41d9b2c4cbeb688c1d26, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/6f6a69c2db2b4958901ceb4a1df25f5e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bd28603c0433492c8e41a9fa7ca59080, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ce61a956c5084d6f96e0841a646b0a10, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/2b91b7ddf8564927baf075f2ae13c7a8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4727c01cbecc4240b66756747b86c822, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0e7f28e39e5643cd919ce143d6579fad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/779a4516dd5c479a80a559f6f1dd64db, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/b6a1da9220144417bf4a9ab101f7c9e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4a9f67773eb4cf7ac9dd678aebdc8a8] to archive 2024-12-09T19:04:55,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:04:55,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/fa7465c7a5da490c9f4c9a44b36ba400 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/fa7465c7a5da490c9f4c9a44b36ba400 2024-12-09T19:04:55,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1686c3da175b4dd1840b01278d5c1e1e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1686c3da175b4dd1840b01278d5c1e1e 2024-12-09T19:04:55,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/d91bcb7329c548a3ac7a6328bb1eeadf to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/d91bcb7329c548a3ac7a6328bb1eeadf 2024-12-09T19:04:55,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4ac258c6e0446cfb78217ee12d9f316 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4ac258c6e0446cfb78217ee12d9f316 2024-12-09T19:04:55,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/76d9df0472b04488a77282192004b6b2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/76d9df0472b04488a77282192004b6b2 2024-12-09T19:04:55,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/83eb7da0ca0c4340bf24b80a6af1640b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/83eb7da0ca0c4340bf24b80a6af1640b 2024-12-09T19:04:55,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/90b140e7bb4a4a2a8b38485873910d74 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/90b140e7bb4a4a2a8b38485873910d74 2024-12-09T19:04:55,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/22b158644f6044f8bf52cacbc58690fb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/22b158644f6044f8bf52cacbc58690fb 2024-12-09T19:04:55,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f93bebfe16a94f52b58fe04515279297 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f93bebfe16a94f52b58fe04515279297 2024-12-09T19:04:55,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0a3c4efa777e486c8ea4212e0acde4e1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0a3c4efa777e486c8ea4212e0acde4e1 2024-12-09T19:04:55,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f97892ca7e224af5b593c66624a70576 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/f97892ca7e224af5b593c66624a70576 2024-12-09T19:04:55,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/58739913a3ee44c28b800f42bd1fdefa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/58739913a3ee44c28b800f42bd1fdefa 2024-12-09T19:04:55,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/9395448e805d4a3fb2e3ddfde804cd34 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/9395448e805d4a3fb2e3ddfde804cd34 2024-12-09T19:04:55,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ddc665425e544e609e8bd70cb496682c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ddc665425e544e609e8bd70cb496682c 2024-12-09T19:04:55,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/42ad896027b748508f522d5bb9bd3718 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/42ad896027b748508f522d5bb9bd3718 2024-12-09T19:04:55,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1695b47337fc41d9b2c4cbeb688c1d26 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/1695b47337fc41d9b2c4cbeb688c1d26 2024-12-09T19:04:55,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/6f6a69c2db2b4958901ceb4a1df25f5e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/6f6a69c2db2b4958901ceb4a1df25f5e 2024-12-09T19:04:55,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bd28603c0433492c8e41a9fa7ca59080 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bd28603c0433492c8e41a9fa7ca59080 2024-12-09T19:04:55,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ce61a956c5084d6f96e0841a646b0a10 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/ce61a956c5084d6f96e0841a646b0a10 2024-12-09T19:04:55,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/2b91b7ddf8564927baf075f2ae13c7a8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/2b91b7ddf8564927baf075f2ae13c7a8 2024-12-09T19:04:55,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4727c01cbecc4240b66756747b86c822 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4727c01cbecc4240b66756747b86c822 2024-12-09T19:04:55,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0e7f28e39e5643cd919ce143d6579fad to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/0e7f28e39e5643cd919ce143d6579fad 2024-12-09T19:04:55,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/779a4516dd5c479a80a559f6f1dd64db to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/779a4516dd5c479a80a559f6f1dd64db 2024-12-09T19:04:55,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/b6a1da9220144417bf4a9ab101f7c9e1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/b6a1da9220144417bf4a9ab101f7c9e1 2024-12-09T19:04:55,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4a9f67773eb4cf7ac9dd678aebdc8a8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/e4a9f67773eb4cf7ac9dd678aebdc8a8 2024-12-09T19:04:55,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/c25523d6b05c4616a5781470cabf7b25, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8b293ed07e724524b858eda58b8db4d2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6ff6f1be9a514c50bc856ce60f049c98, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/595a4df6b81a446790ffa26437f53da6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6799a237693e47ec9563703c881a1637, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1d5d259866744d0cab6d13cde0e44a98, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/880a567de55141a7b7ae93a58710dd84, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/598d09c3686044f79e7d7d67c483fc83, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/944dd1b0e32c43a494885b6027c05c2e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eec2628a5ab142b1994286e26e70e346, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/de5ea38bc6b34188a68ae9f0d6349403, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2f88829d7c3b4e27b566a11fbaf51594, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8619e7b0cdf6475fa3a7822c4cee5e1e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eb5a67eb03984831bd635eca9735194c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/3a717dcb49ed45f7adb2e697a377844e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/4ecf1dbda5b94f949dc193fb5b5408c6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1c63624e5ded4a36aeae56d9dae9987c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/9e5f3f6a77614ddcb4441922f81e8c35, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/a1f26cead2bd4b27bd4a9b15dca97546, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/adced595fbaa4da0a174cbe2a8c7ef1d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/ecc4018bbe204672baa5f9ad4a2ac7d1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/39fff0b47a254b0fbe7e33fe2e6a2ad9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/97a9a72a47654402ba90fdc82bac28dd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1e4ddb3c0f554eeeb486275371902b28] to archive 2024-12-09T19:04:55,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:04:55,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/c25523d6b05c4616a5781470cabf7b25 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/c25523d6b05c4616a5781470cabf7b25 2024-12-09T19:04:55,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8b293ed07e724524b858eda58b8db4d2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8b293ed07e724524b858eda58b8db4d2 2024-12-09T19:04:55,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6ff6f1be9a514c50bc856ce60f049c98 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6ff6f1be9a514c50bc856ce60f049c98 2024-12-09T19:04:55,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/595a4df6b81a446790ffa26437f53da6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/595a4df6b81a446790ffa26437f53da6 2024-12-09T19:04:55,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6799a237693e47ec9563703c881a1637 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/6799a237693e47ec9563703c881a1637 2024-12-09T19:04:55,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1d5d259866744d0cab6d13cde0e44a98 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1d5d259866744d0cab6d13cde0e44a98 2024-12-09T19:04:55,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/880a567de55141a7b7ae93a58710dd84 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/880a567de55141a7b7ae93a58710dd84 2024-12-09T19:04:55,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/598d09c3686044f79e7d7d67c483fc83 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/598d09c3686044f79e7d7d67c483fc83 2024-12-09T19:04:55,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/944dd1b0e32c43a494885b6027c05c2e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/944dd1b0e32c43a494885b6027c05c2e 2024-12-09T19:04:55,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eec2628a5ab142b1994286e26e70e346 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eec2628a5ab142b1994286e26e70e346 2024-12-09T19:04:55,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/de5ea38bc6b34188a68ae9f0d6349403 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/de5ea38bc6b34188a68ae9f0d6349403 2024-12-09T19:04:55,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2f88829d7c3b4e27b566a11fbaf51594 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2f88829d7c3b4e27b566a11fbaf51594 2024-12-09T19:04:55,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8619e7b0cdf6475fa3a7822c4cee5e1e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/8619e7b0cdf6475fa3a7822c4cee5e1e 2024-12-09T19:04:55,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eb5a67eb03984831bd635eca9735194c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/eb5a67eb03984831bd635eca9735194c 2024-12-09T19:04:55,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/3a717dcb49ed45f7adb2e697a377844e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/3a717dcb49ed45f7adb2e697a377844e 2024-12-09T19:04:55,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/4ecf1dbda5b94f949dc193fb5b5408c6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/4ecf1dbda5b94f949dc193fb5b5408c6 2024-12-09T19:04:55,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1c63624e5ded4a36aeae56d9dae9987c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1c63624e5ded4a36aeae56d9dae9987c 2024-12-09T19:04:55,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/9e5f3f6a77614ddcb4441922f81e8c35 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/9e5f3f6a77614ddcb4441922f81e8c35 2024-12-09T19:04:55,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/a1f26cead2bd4b27bd4a9b15dca97546 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/a1f26cead2bd4b27bd4a9b15dca97546 2024-12-09T19:04:55,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f78b9f1e5c1640c5a2e1d3b0060a5b1e 2024-12-09T19:04:55,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/adced595fbaa4da0a174cbe2a8c7ef1d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/adced595fbaa4da0a174cbe2a8c7ef1d 2024-12-09T19:04:55,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/ecc4018bbe204672baa5f9ad4a2ac7d1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/ecc4018bbe204672baa5f9ad4a2ac7d1 2024-12-09T19:04:55,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/39fff0b47a254b0fbe7e33fe2e6a2ad9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/39fff0b47a254b0fbe7e33fe2e6a2ad9 2024-12-09T19:04:55,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/97a9a72a47654402ba90fdc82bac28dd to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/97a9a72a47654402ba90fdc82bac28dd 2024-12-09T19:04:55,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1e4ddb3c0f554eeeb486275371902b28 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/1e4ddb3c0f554eeeb486275371902b28 2024-12-09T19:04:55,119 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-12-09T19:04:55,119 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. 2024-12-09T19:04:55,119 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for f95442f56e862cbd2c6f1b449c5bb8c9: Waiting for close lock at 1733771093761Running coprocessor pre-close hooks at 1733771093761Disabling compacts and flushes for region at 1733771093761Disabling writes for close at 1733771093761Obtaining lock to block concurrent updates at 1733771093762 (+1 ms)Preparing flush snapshotting stores in f95442f56e862cbd2c6f1b449c5bb8c9 at 1733771093762Finished memstore snapshotting TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733771093762Flushing stores of TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9. at 1733771093764 (+2 ms)Flushing f95442f56e862cbd2c6f1b449c5bb8c9/A: creating writer at 1733771093764Flushing f95442f56e862cbd2c6f1b449c5bb8c9/A: appending metadata at 1733771093771 (+7 ms)Flushing f95442f56e862cbd2c6f1b449c5bb8c9/A: closing flushed file at 1733771093771Flushing f95442f56e862cbd2c6f1b449c5bb8c9/B: creating writer at 1733771094188 (+417 ms)Flushing f95442f56e862cbd2c6f1b449c5bb8c9/B: appending metadata at 1733771094190 (+2 ms)Flushing f95442f56e862cbd2c6f1b449c5bb8c9/B: closing flushed file at 1733771094190Flushing f95442f56e862cbd2c6f1b449c5bb8c9/C: creating writer at 1733771094604 (+414 ms)Flushing f95442f56e862cbd2c6f1b449c5bb8c9/C: appending metadata at 1733771094608 (+4 ms)Flushing f95442f56e862cbd2c6f1b449c5bb8c9/C: closing flushed file at 1733771094608Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3319431: reopening flushed file at 1733771095022 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60276bd9: reopening flushed file at 1733771095029 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c976d6: reopening flushed file at 1733771095036 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f95442f56e862cbd2c6f1b449c5bb8c9 in 1280ms, sequenceid=373, compaction requested=true at 1733771095041 (+5 ms)Writing region close event to WAL at 1733771095116 (+75 ms)Running coprocessor post-close hooks at 1733771095119 (+3 ms)Closed at 1733771095119 2024-12-09T19:04:55,121 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:55,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=f95442f56e862cbd2c6f1b449c5bb8c9, regionState=CLOSED 2024-12-09T19:04:55,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:04:55,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T19:04:55,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure f95442f56e862cbd2c6f1b449c5bb8c9, server=a437f8b9ba7d,46189,1733770998237 in 1.5160 sec 2024-12-09T19:04:55,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-09T19:04:55,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f95442f56e862cbd2c6f1b449c5bb8c9, UNASSIGN in 1.5210 sec 2024-12-09T19:04:55,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T19:04:55,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5230 sec 2024-12-09T19:04:55,127 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771095127"}]},"ts":"1733771095127"} 2024-12-09T19:04:55,128 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T19:04:55,128 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T19:04:55,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-12-09T19:04:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T19:04:55,756 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:55,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T19:04:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,761 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T19:04:55,763 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,766 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:55,770 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/recovered.edits] 2024-12-09T19:04:55,775 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/816336e1aff648478e298e504f2f090b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/816336e1aff648478e298e504f2f090b 2024-12-09T19:04:55,778 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/dd975a840f3e44a9a54e6bd437165adc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/dd975a840f3e44a9a54e6bd437165adc 2024-12-09T19:04:55,780 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/e913528b9cb84cd1b45b049d04c49250 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/A/e913528b9cb84cd1b45b049d04c49250 2024-12-09T19:04:55,784 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4ec06b3626664fc4a4fe829e29e9f342 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/4ec06b3626664fc4a4fe829e29e9f342 2024-12-09T19:04:55,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bfecffe672c34ac6823ec2f8071beff8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/bfecffe672c34ac6823ec2f8071beff8 2024-12-09T19:04:55,788 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/c2f693b3445e4d9294c388beb27efa2a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/B/c2f693b3445e4d9294c388beb27efa2a 2024-12-09T19:04:55,792 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/27e8bc52538d4204a3cd43e910703ac9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/27e8bc52538d4204a3cd43e910703ac9 2024-12-09T19:04:55,794 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2c2034ba970844a9b819554fe93a5d2e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/2c2034ba970844a9b819554fe93a5d2e 2024-12-09T19:04:55,796 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f14c72c3b2d04dedaa607fe154245b23 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/C/f14c72c3b2d04dedaa607fe154245b23 2024-12-09T19:04:55,800 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/recovered.edits/376.seqid to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9/recovered.edits/376.seqid 2024-12-09T19:04:55,801 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/f95442f56e862cbd2c6f1b449c5bb8c9 2024-12-09T19:04:55,801 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T19:04:55,804 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,807 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T19:04:55,809 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T19:04:55,810 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,810 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T19:04:55,810 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733771095810"}]},"ts":"9223372036854775807"} 2024-12-09T19:04:55,812 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T19:04:55,812 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f95442f56e862cbd2c6f1b449c5bb8c9, NAME => 'TestAcidGuarantees,,1733771062570.f95442f56e862cbd2c6f1b449c5bb8c9.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T19:04:55,812 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T19:04:55,812 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733771095812"}]},"ts":"9223372036854775807"} 2024-12-09T19:04:55,814 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T19:04:55,814 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-09T19:04:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T19:04:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T19:04:55,865 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:55,876 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 242), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=2202 (was 2229) 2024-12-09T19:04:55,886 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=2202 2024-12-09T19:04:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:04:55,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:04:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:55,890 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T19:04:55,890 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:55,890 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-09T19:04:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T19:04:55,891 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T19:04:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742117_1293 (size=959) 2024-12-09T19:04:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T19:04:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T19:04:56,302 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:04:56,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742118_1294 (size=53) 2024-12-09T19:04:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T19:04:56,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:04:56,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5ae3551814f4d0fc673941d40ec716c2, disabling compactions & flushes 2024-12-09T19:04:56,735 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:56,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:56,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. after waiting 0 ms 2024-12-09T19:04:56,735 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:56,736 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:56,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5ae3551814f4d0fc673941d40ec716c2: Waiting for close lock at 1733771096735Disabling compacts and flushes for region at 1733771096735Disabling writes for close at 1733771096735Writing region close event to WAL at 1733771096735Closed at 1733771096735 2024-12-09T19:04:56,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T19:04:56,740 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733771096739"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733771096739"}]},"ts":"1733771096739"} 2024-12-09T19:04:56,742 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T19:04:56,743 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T19:04:56,743 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771096743"}]},"ts":"1733771096743"} 2024-12-09T19:04:56,744 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T19:04:56,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, ASSIGN}] 2024-12-09T19:04:56,746 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, ASSIGN 2024-12-09T19:04:56,746 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=false 2024-12-09T19:04:56,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:56,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, ASSIGN because future has completed 2024-12-09T19:04:56,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:04:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T19:04:57,061 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:57,061 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:04:57,061 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,061 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:04:57,062 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,062 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,064 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,065 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae3551814f4d0fc673941d40ec716c2 columnFamilyName A 2024-12-09T19:04:57,065 DEBUG [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:57,066 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(327): Store=5ae3551814f4d0fc673941d40ec716c2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:57,066 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,067 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae3551814f4d0fc673941d40ec716c2 columnFamilyName B 2024-12-09T19:04:57,067 DEBUG [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:57,067 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(327): Store=5ae3551814f4d0fc673941d40ec716c2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:57,067 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,069 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae3551814f4d0fc673941d40ec716c2 columnFamilyName C 2024-12-09T19:04:57,069 DEBUG [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:57,069 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(327): Store=5ae3551814f4d0fc673941d40ec716c2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:57,069 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,070 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,070 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,072 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,072 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,072 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:04:57,073 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,076 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:04:57,076 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 5ae3551814f4d0fc673941d40ec716c2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64577986, jitterRate=-0.03771302103996277}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:04:57,076 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:57,077 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 5ae3551814f4d0fc673941d40ec716c2: Running coprocessor pre-open hook at 1733771097062Writing region info on filesystem at 1733771097062Initializing all the Stores at 1733771097063 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771097063Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771097063Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771097063Cleaning up temporary data from old regions at 1733771097072 (+9 ms)Running coprocessor post-open hooks at 1733771097076 (+4 ms)Region opened successfully at 1733771097077 (+1 ms) 2024-12-09T19:04:57,078 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., pid=85, masterSystemTime=1733771097055 2024-12-09T19:04:57,081 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:57,081 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:57,082 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:57,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:04:57,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-09T19:04:57,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 in 183 msec 2024-12-09T19:04:57,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-09T19:04:57,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, ASSIGN in 342 msec 2024-12-09T19:04:57,089 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T19:04:57,089 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771097089"}]},"ts":"1733771097089"} 2024-12-09T19:04:57,090 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T19:04:57,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T19:04:57,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2040 sec 2024-12-09T19:04:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-09T19:04:58,046 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:58,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:58,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:58,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:58,053 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:58,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1bb468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:58,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:58,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398bf7cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:58,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:58,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:58,058 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:04:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:04:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47cce5a6 2024-12-09T19:04:58,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:04:58,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:04:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:04:58,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:04:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T19:04:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742119_1295 (size=995) 2024-12-09T19:04:58,476 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-09T19:04:58,476 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-09T19:04:58,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:04:58,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, REOPEN/MOVE}] 2024-12-09T19:04:58,487 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, REOPEN/MOVE 2024-12-09T19:04:58,488 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:58,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, REOPEN/MOVE because future has completed 2024-12-09T19:04:58,489 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:04:58,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:04:58,643 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,643 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:04:58,644 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 5ae3551814f4d0fc673941d40ec716c2, disabling compactions & flushes 2024-12-09T19:04:58,644 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:58,644 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:58,644 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. after waiting 0 ms 2024-12-09T19:04:58,644 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:58,652 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T19:04:58,652 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:58,652 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 5ae3551814f4d0fc673941d40ec716c2: Waiting for close lock at 1733771098644Running coprocessor pre-close hooks at 1733771098644Disabling compacts and flushes for region at 1733771098644Disabling writes for close at 1733771098644Writing region close event to WAL at 1733771098645 (+1 ms)Running coprocessor post-close hooks at 1733771098652 (+7 ms)Closed at 1733771098652 2024-12-09T19:04:58,652 WARN [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 5ae3551814f4d0fc673941d40ec716c2 to self. 2024-12-09T19:04:58,654 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=CLOSED 2024-12-09T19:04:58,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:04:58,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-09T19:04:58,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 in 167 msec 2024-12-09T19:04:58,659 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, REOPEN/MOVE; state=CLOSED, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=true 2024-12-09T19:04:58,810 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:58,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, REOPEN/MOVE because future has completed 2024-12-09T19:04:58,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:04:58,975 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:58,975 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:04:58,975 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,976 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:04:58,976 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,976 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,978 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,980 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae3551814f4d0fc673941d40ec716c2 columnFamilyName A 2024-12-09T19:04:58,982 DEBUG [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:58,983 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(327): Store=5ae3551814f4d0fc673941d40ec716c2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:58,983 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,985 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae3551814f4d0fc673941d40ec716c2 columnFamilyName B 2024-12-09T19:04:58,985 DEBUG [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:58,985 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(327): Store=5ae3551814f4d0fc673941d40ec716c2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:58,986 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,987 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ae3551814f4d0fc673941d40ec716c2 columnFamilyName C 2024-12-09T19:04:58,987 DEBUG [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:58,987 INFO [StoreOpener-5ae3551814f4d0fc673941d40ec716c2-1 {}] regionserver.HStore(327): Store=5ae3551814f4d0fc673941d40ec716c2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:04:58,987 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,989 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,990 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,992 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,992 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,993 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:04:58,995 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,997 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 5ae3551814f4d0fc673941d40ec716c2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72428407, jitterRate=0.07926736772060394}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:04:58,997 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:58,998 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 5ae3551814f4d0fc673941d40ec716c2: Running coprocessor pre-open hook at 1733771098976Writing region info on filesystem at 1733771098976Initializing all the Stores at 1733771098977 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771098977Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771098978 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771098978Cleaning up temporary data from old regions at 1733771098992 (+14 ms)Running coprocessor post-open hooks at 1733771098997 (+5 ms)Region opened successfully at 1733771098998 (+1 ms) 2024-12-09T19:04:58,999 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., pid=90, masterSystemTime=1733771098970 2024-12-09T19:04:59,001 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,001 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,001 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=OPEN, openSeqNum=5, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:04:59,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-09T19:04:59,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 in 189 msec 2024-12-09T19:04:59,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T19:04:59,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, REOPEN/MOVE in 521 msec 2024-12-09T19:04:59,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T19:04:59,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 529 msec 2024-12-09T19:04:59,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 953 msec 2024-12-09T19:04:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T19:04:59,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:04:59,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,021 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195b0a10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f6e9a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,025 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a349cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9cb95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593c9473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b81f11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,034 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf53d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,035 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142a1636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,037 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e234a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed29c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,041 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e606c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,043 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77442c37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,045 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e784ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3f6f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca81dbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a5ce2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b812c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bad041b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:04:59,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:04:59,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:04:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:04:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:04:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbba6d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:04:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:04:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:04:59,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:04:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7a034a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:04:59,060 DEBUG [Thread-1384 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,060 DEBUG [Thread-1371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,060 DEBUG [Thread-1386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,060 DEBUG [Thread-1375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,061 DEBUG [Thread-1390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,061 DEBUG [Thread-1379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,062 DEBUG [Thread-1382 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,062 DEBUG [Thread-1377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,062 DEBUG [Thread-1373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,062 DEBUG [Thread-1388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:04:59,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:04:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-09T19:04:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:04:59,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:04:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:04:59,067 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:04:59,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:04:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,068 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:04:59,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:04:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:04:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:59,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:04:59,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:04:59,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091c3abda85bcc4fb8a18352b11d235245_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771099077/Put/seqid=0 2024-12-09T19:04:59,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771159100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771159101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771159102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742120_1296 (size=17034) 2024-12-09T19:04:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771159105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771159106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:04:59,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771159214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771159214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771159214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771159215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771159215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:04:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:04:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:04:59,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:04:59,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:04:59,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771159424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771159425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771159425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771159425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771159425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:04:59,515 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091c3abda85bcc4fb8a18352b11d235245_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091c3abda85bcc4fb8a18352b11d235245_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:04:59,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/9ad9cd6a2d984f41af1bd7f9f8603014, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:04:59,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/9ad9cd6a2d984f41af1bd7f9f8603014 is 175, key is test_row_0/A:col10/1733771099077/Put/seqid=0 2024-12-09T19:04:59,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742121_1297 (size=48139) 2024-12-09T19:04:59,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:04:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:04:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:04:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:04:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:04:59,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771159734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771159734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771159735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771159735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771159735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:04:59,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:04:59,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:04:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:04:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/9ad9cd6a2d984f41af1bd7f9f8603014 2024-12-09T19:04:59,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/b3aabd76dcb4419b98892ae46ca29935 is 50, key is test_row_0/B:col10/1733771099077/Put/seqid=0 2024-12-09T19:04:59,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742122_1298 (size=12001) 2024-12-09T19:04:59,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:04:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:04:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:04:59,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:04:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:05:00,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:00,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:05:00,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771160244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771160244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771160244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771160245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771160245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:00,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:00,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:00,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:05:00,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:00,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/b3aabd76dcb4419b98892ae46ca29935 2024-12-09T19:05:00,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/c78558ff5583474abc5ca7d7f633f438 is 50, key is test_row_0/C:col10/1733771099077/Put/seqid=0 2024-12-09T19:05:00,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742123_1299 (size=12001) 2024-12-09T19:05:00,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:05:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:05:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,643 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:05:00,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:05:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:00,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/c78558ff5583474abc5ca7d7f633f438 2024-12-09T19:05:00,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/9ad9cd6a2d984f41af1bd7f9f8603014 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014 2024-12-09T19:05:00,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014, entries=250, sequenceid=19, filesize=47.0 K 2024-12-09T19:05:00,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/b3aabd76dcb4419b98892ae46ca29935 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b3aabd76dcb4419b98892ae46ca29935 2024-12-09T19:05:00,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b3aabd76dcb4419b98892ae46ca29935, entries=150, sequenceid=19, filesize=11.7 K 2024-12-09T19:05:00,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/c78558ff5583474abc5ca7d7f633f438 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/c78558ff5583474abc5ca7d7f633f438 2024-12-09T19:05:00,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/c78558ff5583474abc5ca7d7f633f438, entries=150, sequenceid=19, filesize=11.7 K 2024-12-09T19:05:00,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5ae3551814f4d0fc673941d40ec716c2 in 1707ms, sequenceid=19, compaction requested=false 2024-12-09T19:05:00,785 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T19:05:00,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:00,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-09T19:05:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:00,903 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-09T19:05:00,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093365071752fe4f3d8fe776a96bb0b68f_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771099104/Put/seqid=0 2024-12-09T19:05:00,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742124_1300 (size=12154) 2024-12-09T19:05:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:05:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:01,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771161271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771161273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771161274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771161277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771161278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:01,341 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093365071752fe4f3d8fe776a96bb0b68f_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093365071752fe4f3d8fe776a96bb0b68f_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/2f76ad73479d49259468b8cd0396c834, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/2f76ad73479d49259468b8cd0396c834 is 175, key is test_row_0/A:col10/1733771099104/Put/seqid=0 2024-12-09T19:05:01,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742125_1301 (size=30955) 2024-12-09T19:05:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771161385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771161385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771161385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771161385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771161394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771161594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771161594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771161594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771161594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771161604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,747 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/2f76ad73479d49259468b8cd0396c834 2024-12-09T19:05:01,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/b97e2ba5282945bdbd5995351384fdd4 is 50, key is test_row_0/B:col10/1733771099104/Put/seqid=0 2024-12-09T19:05:01,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742126_1302 (size=12001) 2024-12-09T19:05:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771161905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771161905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771161905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771161905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771161914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:02,158 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/b97e2ba5282945bdbd5995351384fdd4 2024-12-09T19:05:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/fb3c53bbd23f45eca69a3a665beb9d84 is 50, key is test_row_0/C:col10/1733771099104/Put/seqid=0 2024-12-09T19:05:02,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742127_1303 (size=12001) 2024-12-09T19:05:02,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771162424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771162424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771162424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771162424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:02,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771162425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:02,567 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/fb3c53bbd23f45eca69a3a665beb9d84 2024-12-09T19:05:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/2f76ad73479d49259468b8cd0396c834 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834 2024-12-09T19:05:02,574 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834, entries=150, sequenceid=40, filesize=30.2 K 2024-12-09T19:05:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/b97e2ba5282945bdbd5995351384fdd4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b97e2ba5282945bdbd5995351384fdd4 2024-12-09T19:05:02,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b97e2ba5282945bdbd5995351384fdd4, entries=150, sequenceid=40, filesize=11.7 K 2024-12-09T19:05:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/fb3c53bbd23f45eca69a3a665beb9d84 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/fb3c53bbd23f45eca69a3a665beb9d84 2024-12-09T19:05:02,581 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/fb3c53bbd23f45eca69a3a665beb9d84, entries=150, sequenceid=40, filesize=11.7 K 2024-12-09T19:05:02,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5ae3551814f4d0fc673941d40ec716c2 in 1679ms, sequenceid=40, compaction requested=false 2024-12-09T19:05:02,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:02,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:02,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-09T19:05:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-09T19:05:02,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T19:05:02,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5150 sec 2024-12-09T19:05:02,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 3.5220 sec 2024-12-09T19:05:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T19:05:03,224 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:03,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-09T19:05:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T19:05:03,226 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:03,227 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:03,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T19:05:03,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-09T19:05:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:03,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-09T19:05:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098483104113684beebb7bcdea05374af9_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771101266/Put/seqid=0 2024-12-09T19:05:03,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742128_1304 (size=12154) 2024-12-09T19:05:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:03,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771163460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771163460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771163461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771163463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771163464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T19:05:03,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771163575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771163575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771163575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771163575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771163575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771163784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771163784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771163784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771163784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771163784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:03,792 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098483104113684beebb7bcdea05374af9_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098483104113684beebb7bcdea05374af9_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/6e01e9cdd7c445a48c2d794b230d3352, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/6e01e9cdd7c445a48c2d794b230d3352 is 175, key is test_row_0/A:col10/1733771101266/Put/seqid=0 2024-12-09T19:05:03,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742129_1305 (size=30955) 2024-12-09T19:05:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T19:05:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771164094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771164094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771164094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771164094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771164104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,198 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/6e01e9cdd7c445a48c2d794b230d3352 2024-12-09T19:05:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/565a6090b79a49f488e2adb74e61f059 is 50, key is test_row_0/B:col10/1733771101266/Put/seqid=0 2024-12-09T19:05:04,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742130_1306 (size=12001) 2024-12-09T19:05:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T19:05:04,608 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/565a6090b79a49f488e2adb74e61f059 2024-12-09T19:05:04,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771164604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771164604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771164604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/19215e0e369a4fc887091bc3010fc07f is 50, key is test_row_0/C:col10/1733771101266/Put/seqid=0 2024-12-09T19:05:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742131_1307 (size=12001) 2024-12-09T19:05:04,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771164614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:04,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771164625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/19215e0e369a4fc887091bc3010fc07f 2024-12-09T19:05:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/6e01e9cdd7c445a48c2d794b230d3352 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352 2024-12-09T19:05:05,025 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352, entries=150, sequenceid=55, filesize=30.2 K 2024-12-09T19:05:05,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/565a6090b79a49f488e2adb74e61f059 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/565a6090b79a49f488e2adb74e61f059 2024-12-09T19:05:05,029 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/565a6090b79a49f488e2adb74e61f059, entries=150, sequenceid=55, filesize=11.7 K 2024-12-09T19:05:05,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/19215e0e369a4fc887091bc3010fc07f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/19215e0e369a4fc887091bc3010fc07f 2024-12-09T19:05:05,034 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/19215e0e369a4fc887091bc3010fc07f, entries=150, sequenceid=55, filesize=11.7 K 2024-12-09T19:05:05,034 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5ae3551814f4d0fc673941d40ec716c2 in 1655ms, sequenceid=55, compaction requested=true 2024-12-09T19:05:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-09T19:05:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-09T19:05:05,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-09T19:05:05,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-12-09T19:05:05,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.8130 sec 2024-12-09T19:05:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-09T19:05:05,374 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:05,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-09T19:05:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T19:05:05,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:05,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:05,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T19:05:05,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-09T19:05:05,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:05,529 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-09T19:05:05,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098e927a79e2054e858d61228d84d211e7_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771103453/Put/seqid=0 2024-12-09T19:05:05,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742132_1308 (size=12154) 2024-12-09T19:05:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:05,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771165644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771165644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,654 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771165651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771165652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771165653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T19:05:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771165764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771165764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771165765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771165765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:05,942 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098e927a79e2054e858d61228d84d211e7_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098e927a79e2054e858d61228d84d211e7_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/c881cdbc09fe4eb8a8f279ca5903e4ab, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/c881cdbc09fe4eb8a8f279ca5903e4ab is 175, key is test_row_0/A:col10/1733771103453/Put/seqid=0 2024-12-09T19:05:05,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742133_1309 (size=30955) 2024-12-09T19:05:05,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771165975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771165975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771165975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771165984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T19:05:06,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771166285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771166285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771166294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771166294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/c881cdbc09fe4eb8a8f279ca5903e4ab 2024-12-09T19:05:06,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/d6516ee3644a4c93aad0e83c32c35ae4 is 50, key is test_row_0/B:col10/1733771103453/Put/seqid=0 2024-12-09T19:05:06,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742134_1310 (size=12001) 2024-12-09T19:05:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T19:05:06,782 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/d6516ee3644a4c93aad0e83c32c35ae4 2024-12-09T19:05:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/3406cea5f28844789ef5ae597f5af791 is 50, key is test_row_0/C:col10/1733771103453/Put/seqid=0 2024-12-09T19:05:06,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742135_1311 (size=12001) 2024-12-09T19:05:06,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771166795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771166795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771166804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771166804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,192 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/3406cea5f28844789ef5ae597f5af791 2024-12-09T19:05:07,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/c881cdbc09fe4eb8a8f279ca5903e4ab as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab 2024-12-09T19:05:07,198 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab, entries=150, sequenceid=76, filesize=30.2 K 2024-12-09T19:05:07,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/d6516ee3644a4c93aad0e83c32c35ae4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/d6516ee3644a4c93aad0e83c32c35ae4 2024-12-09T19:05:07,203 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/d6516ee3644a4c93aad0e83c32c35ae4, entries=150, sequenceid=76, filesize=11.7 K 2024-12-09T19:05:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/3406cea5f28844789ef5ae597f5af791 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3406cea5f28844789ef5ae597f5af791 2024-12-09T19:05:07,207 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3406cea5f28844789ef5ae597f5af791, entries=150, sequenceid=76, filesize=11.7 K 2024-12-09T19:05:07,208 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5ae3551814f4d0fc673941d40ec716c2 in 1679ms, sequenceid=76, compaction requested=true 2024-12-09T19:05:07,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:07,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T19:05:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-09T19:05:07,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T19:05:07,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-12-09T19:05:07,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8370 sec 2024-12-09T19:05:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-09T19:05:07,525 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:07,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-09T19:05:07,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:07,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:07,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:07,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:07,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-09T19:05:07,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209024d6b5d72ba48159e2b2faaa67a9754_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:07,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742136_1312 (size=14594) 2024-12-09T19:05:07,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771167786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771167815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771167815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:07,821 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771167815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,821 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,821 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771167824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:07,830 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:07,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771167904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:07,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:07,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,078 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:08,082 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209024d6b5d72ba48159e2b2faaa67a9754_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209024d6b5d72ba48159e2b2faaa67a9754_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:08,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/057fdf0d300346759f027b4f5f379d7c, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:08,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/057fdf0d300346759f027b4f5f379d7c is 175, key is test_row_0/A:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:08,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742137_1313 (size=39549) 2024-12-09T19:05:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771168115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:08,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:08,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771168424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:08,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:08,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/057fdf0d300346759f027b4f5f379d7c 2024-12-09T19:05:08,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/639f953377034ce5a1577cd376d312e7 is 50, key is test_row_0/B:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:08,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742138_1314 (size=12001) 2024-12-09T19:05:08,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/639f953377034ce5a1577cd376d312e7 2024-12-09T19:05:08,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/8e83110a24e4473081f1605c429ee008 is 50, key is test_row_0/C:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:08,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742139_1315 (size=12001) 2024-12-09T19:05:08,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:08,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:08,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/8e83110a24e4473081f1605c429ee008 2024-12-09T19:05:08,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/057fdf0d300346759f027b4f5f379d7c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c 2024-12-09T19:05:08,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c, entries=200, sequenceid=92, filesize=38.6 K 2024-12-09T19:05:08,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/639f953377034ce5a1577cd376d312e7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/639f953377034ce5a1577cd376d312e7 2024-12-09T19:05:08,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/639f953377034ce5a1577cd376d312e7, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:05:08,917 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-09T19:05:08,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/8e83110a24e4473081f1605c429ee008 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/8e83110a24e4473081f1605c429ee008 2024-12-09T19:05:08,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/8e83110a24e4473081f1605c429ee008, entries=150, sequenceid=92, filesize=11.7 K 2024-12-09T19:05:08,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5ae3551814f4d0fc673941d40ec716c2 in 1252ms, sequenceid=92, compaction requested=true 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:08,921 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:08,921 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:08,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 180553 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/A is initiating minor compaction (all files) 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/B is initiating minor compaction (all files) 2024-12-09T19:05:08,923 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/B in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,923 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/A in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,923 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b3aabd76dcb4419b98892ae46ca29935, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b97e2ba5282945bdbd5995351384fdd4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/565a6090b79a49f488e2adb74e61f059, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/d6516ee3644a4c93aad0e83c32c35ae4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/639f953377034ce5a1577cd376d312e7] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=58.6 K 2024-12-09T19:05:08,923 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=176.3 K 2024-12-09T19:05:08,923 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c] 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b3aabd76dcb4419b98892ae46ca29935, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1733771099076 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad9cd6a2d984f41af1bd7f9f8603014, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1733771099073 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b97e2ba5282945bdbd5995351384fdd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733771099099 2024-12-09T19:05:08,923 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f76ad73479d49259468b8cd0396c834, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733771099099 2024-12-09T19:05:08,924 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 565a6090b79a49f488e2adb74e61f059, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733771101266 2024-12-09T19:05:08,924 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e01e9cdd7c445a48c2d794b230d3352, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733771101266 2024-12-09T19:05:08,924 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting d6516ee3644a4c93aad0e83c32c35ae4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733771103453 2024-12-09T19:05:08,924 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c881cdbc09fe4eb8a8f279ca5903e4ab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733771103453 2024-12-09T19:05:08,925 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057fdf0d300346759f027b4f5f379d7c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771105641 2024-12-09T19:05:08,925 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 639f953377034ce5a1577cd376d312e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771105641 2024-12-09T19:05:08,931 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:08,934 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:08,935 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/76ca2d90f81940e090005a1fb17c3e74 is 50, key is test_row_0/B:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:08,936 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412090c48f7d7d88c48ab8f03c72f7abff116_5ae3551814f4d0fc673941d40ec716c2 store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:08,938 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412090c48f7d7d88c48ab8f03c72f7abff116_5ae3551814f4d0fc673941d40ec716c2, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:08,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742140_1316 (size=12173) 2024-12-09T19:05:08,938 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412090c48f7d7d88c48ab8f03c72f7abff116_5ae3551814f4d0fc673941d40ec716c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:08,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:08,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-09T19:05:08,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742141_1317 (size=4469) 2024-12-09T19:05:08,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120914bb32ebab4e47a09dfe57697940e4e0_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771107785/Put/seqid=0 2024-12-09T19:05:08,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742142_1318 (size=14594) 2024-12-09T19:05:09,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771169061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771169174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,342 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/76ca2d90f81940e090005a1fb17c3e74 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/76ca2d90f81940e090005a1fb17c3e74 2024-12-09T19:05:09,347 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/B of 5ae3551814f4d0fc673941d40ec716c2 into 76ca2d90f81940e090005a1fb17c3e74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:09,347 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:09,347 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/B, priority=11, startTime=1733771108921; duration=0sec 2024-12-09T19:05:09,347 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:09,347 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:B 2024-12-09T19:05:09,347 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T19:05:09,348 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T19:05:09,348 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/C is initiating minor compaction (all files) 2024-12-09T19:05:09,348 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/C in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,348 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/c78558ff5583474abc5ca7d7f633f438, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/fb3c53bbd23f45eca69a3a665beb9d84, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/19215e0e369a4fc887091bc3010fc07f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3406cea5f28844789ef5ae597f5af791, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/8e83110a24e4473081f1605c429ee008] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=58.6 K 2024-12-09T19:05:09,349 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting c78558ff5583474abc5ca7d7f633f438, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1733771099076 2024-12-09T19:05:09,349 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting fb3c53bbd23f45eca69a3a665beb9d84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733771099099 2024-12-09T19:05:09,349 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 19215e0e369a4fc887091bc3010fc07f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733771101266 2024-12-09T19:05:09,349 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#A#compaction#267 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:09,350 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/752a67358926490caadfdffca496b99f is 175, key is test_row_0/A:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:09,351 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 3406cea5f28844789ef5ae597f5af791, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733771103453 2024-12-09T19:05:09,351 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e83110a24e4473081f1605c429ee008, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771105641 2024-12-09T19:05:09,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742143_1319 (size=31127) 2024-12-09T19:05:09,365 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/752a67358926490caadfdffca496b99f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/752a67358926490caadfdffca496b99f 2024-12-09T19:05:09,366 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:09,370 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/A of 5ae3551814f4d0fc673941d40ec716c2 into 752a67358926490caadfdffca496b99f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:09,370 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:09,370 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/A, priority=11, startTime=1733771108921; duration=0sec 2024-12-09T19:05:09,370 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:09,370 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:A 2024-12-09T19:05:09,373 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:09,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120914bb32ebab4e47a09dfe57697940e4e0_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120914bb32ebab4e47a09dfe57697940e4e0_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:09,374 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/3b50ffe4f576486cb91e8d1f8595dff0 is 50, key is test_row_0/C:col10/1733771107668/Put/seqid=0 2024-12-09T19:05:09,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/ea60146525cf4b5298f283efc02ab56a, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:09,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/ea60146525cf4b5298f283efc02ab56a is 175, key is test_row_0/A:col10/1733771107785/Put/seqid=0 2024-12-09T19:05:09,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742145_1321 (size=39549) 2024-12-09T19:05:09,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742144_1320 (size=12173) 2024-12-09T19:05:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771169385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:09,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771169694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,779 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/ea60146525cf4b5298f283efc02ab56a 2024-12-09T19:05:09,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/0243d6f623024692ad6b2863320b94c9 is 50, key is test_row_0/B:col10/1733771107785/Put/seqid=0 2024-12-09T19:05:09,790 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/3b50ffe4f576486cb91e8d1f8595dff0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3b50ffe4f576486cb91e8d1f8595dff0 2024-12-09T19:05:09,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742146_1322 (size=12001) 2024-12-09T19:05:09,794 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/C of 5ae3551814f4d0fc673941d40ec716c2 into 3b50ffe4f576486cb91e8d1f8595dff0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:09,794 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:09,794 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/C, priority=11, startTime=1733771108921; duration=0sec 2024-12-09T19:05:09,794 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:09,794 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:C 2024-12-09T19:05:09,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/0243d6f623024692ad6b2863320b94c9 2024-12-09T19:05:09,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/77a4ffb428a1429989ec9468a63789d6 is 50, key is test_row_0/C:col10/1733771107785/Put/seqid=0 2024-12-09T19:05:09,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742147_1323 (size=12001) 2024-12-09T19:05:09,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/77a4ffb428a1429989ec9468a63789d6 2024-12-09T19:05:09,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/ea60146525cf4b5298f283efc02ab56a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a 2024-12-09T19:05:09,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a, entries=200, sequenceid=113, filesize=38.6 K 2024-12-09T19:05:09,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/0243d6f623024692ad6b2863320b94c9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/0243d6f623024692ad6b2863320b94c9 2024-12-09T19:05:09,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771169835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771169835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/0243d6f623024692ad6b2863320b94c9, entries=150, sequenceid=113, filesize=11.7 K 2024-12-09T19:05:09,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771169835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,837 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:09,837 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,839 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/77a4ffb428a1429989ec9468a63789d6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/77a4ffb428a1429989ec9468a63789d6 2024-12-09T19:05:09,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/77a4ffb428a1429989ec9468a63789d6, entries=150, sequenceid=113, filesize=11.7 K 2024-12-09T19:05:09,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5ae3551814f4d0fc673941d40ec716c2 in 905ms, sequenceid=113, compaction requested=false 2024-12-09T19:05:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:09,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:09,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-09T19:05:09,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209850876e211b44c208c46dca882b2b63c_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:09,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742148_1324 (size=12204) 2024-12-09T19:05:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771169957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:09,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:09,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:09,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771170074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:10,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:10,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:10,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771170204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:10,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:10,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209850876e211b44c208c46dca882b2b63c_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209850876e211b44c208c46dca882b2b63c_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:10,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/816d32df6771438884219f381526e44b, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:10,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/816d32df6771438884219f381526e44b is 175, key is test_row_0/A:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:10,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742149_1325 (size=31005) 2024-12-09T19:05:10,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771170284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:10,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771170594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:10,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/816d32df6771438884219f381526e44b 2024-12-09T19:05:10,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/e493c538652d42e0a88a25b3b68d7629 is 50, key is test_row_0/B:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:10,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742150_1326 (size=12051) 2024-12-09T19:05:10,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:10,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/e493c538652d42e0a88a25b3b68d7629 2024-12-09T19:05:11,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/2c5cfa8608f24b13833890f5887efbab is 50, key is test_row_0/C:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:11,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742151_1327 (size=12051) 2024-12-09T19:05:11,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771171104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:11,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771171224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:11,230 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:11,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:11,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:11,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:11,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/2c5cfa8608f24b13833890f5887efbab 2024-12-09T19:05:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/816d32df6771438884219f381526e44b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b 2024-12-09T19:05:11,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b, entries=150, sequenceid=132, filesize=30.3 K 2024-12-09T19:05:11,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/e493c538652d42e0a88a25b3b68d7629 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/e493c538652d42e0a88a25b3b68d7629 2024-12-09T19:05:11,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/e493c538652d42e0a88a25b3b68d7629, entries=150, sequenceid=132, filesize=11.8 K 2024-12-09T19:05:11,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/2c5cfa8608f24b13833890f5887efbab as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/2c5cfa8608f24b13833890f5887efbab 2024-12-09T19:05:11,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/2c5cfa8608f24b13833890f5887efbab, entries=150, sequenceid=132, filesize=11.8 K 2024-12-09T19:05:11,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5ae3551814f4d0fc673941d40ec716c2 in 1672ms, sequenceid=132, compaction requested=true 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:11,522 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:11,522 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:11,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:11,523 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:11,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101681 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:11,523 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/B is initiating minor compaction (all files) 2024-12-09T19:05:11,523 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/A is initiating minor compaction (all files) 2024-12-09T19:05:11,523 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/A in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,523 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/B in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,523 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/76ca2d90f81940e090005a1fb17c3e74, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/0243d6f623024692ad6b2863320b94c9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/e493c538652d42e0a88a25b3b68d7629] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=35.4 K 2024-12-09T19:05:11,523 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/752a67358926490caadfdffca496b99f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=99.3 K 2024-12-09T19:05:11,524 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,524 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/752a67358926490caadfdffca496b99f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b] 2024-12-09T19:05:11,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ca2d90f81940e090005a1fb17c3e74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771105641 2024-12-09T19:05:11,524 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 752a67358926490caadfdffca496b99f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771105641 2024-12-09T19:05:11,524 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0243d6f623024692ad6b2863320b94c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733771107782 2024-12-09T19:05:11,524 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea60146525cf4b5298f283efc02ab56a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733771107775 2024-12-09T19:05:11,525 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e493c538652d42e0a88a25b3b68d7629, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771109035 2024-12-09T19:05:11,525 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 816d32df6771438884219f381526e44b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771109035 2024-12-09T19:05:11,529 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:11,531 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:11,532 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/bdc1479393994d1eab22a3f4331c13b4 is 50, key is test_row_0/B:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:11,534 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209eea605422d344bdd985d815aa1da43ed_5ae3551814f4d0fc673941d40ec716c2 store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:11,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209eea605422d344bdd985d815aa1da43ed_5ae3551814f4d0fc673941d40ec716c2, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:11,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209eea605422d344bdd985d815aa1da43ed_5ae3551814f4d0fc673941d40ec716c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:11,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742152_1328 (size=12325) 2024-12-09T19:05:11,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742153_1329 (size=4469) 2024-12-09T19:05:11,543 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/bdc1479393994d1eab22a3f4331c13b4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/bdc1479393994d1eab22a3f4331c13b4 2024-12-09T19:05:11,548 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/B of 5ae3551814f4d0fc673941d40ec716c2 into bdc1479393994d1eab22a3f4331c13b4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:11,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:11,548 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/B, priority=13, startTime=1733771111522; duration=0sec 2024-12-09T19:05:11,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:11,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:B 2024-12-09T19:05:11,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:11,549 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:11,549 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/C is initiating minor compaction (all files) 2024-12-09T19:05:11,549 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/C in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,549 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3b50ffe4f576486cb91e8d1f8595dff0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/77a4ffb428a1429989ec9468a63789d6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/2c5cfa8608f24b13833890f5887efbab] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=35.4 K 2024-12-09T19:05:11,549 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b50ffe4f576486cb91e8d1f8595dff0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733771105641 2024-12-09T19:05:11,550 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 77a4ffb428a1429989ec9468a63789d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733771107782 2024-12-09T19:05:11,550 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c5cfa8608f24b13833890f5887efbab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771109035 2024-12-09T19:05:11,555 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#C#compaction#278 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:11,555 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/9b37900aa35a4e518b68c3d46e964e8f is 50, key is test_row_0/C:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:11,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742154_1330 (size=12325) 2024-12-09T19:05:11,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-09T19:05:11,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:11,647 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-09T19:05:11,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8805b8ea85d45138d891d9495baee38_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771109940/Put/seqid=0 2024-12-09T19:05:11,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742155_1331 (size=12304) 2024-12-09T19:05:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:11,941 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#A#compaction#276 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:11,941 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/b62133903fe646ea8de49bbb8ba80220 is 175, key is test_row_0/A:col10/1733771109849/Put/seqid=0 2024-12-09T19:05:11,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742156_1332 (size=31279) 2024-12-09T19:05:11,964 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/9b37900aa35a4e518b68c3d46e964e8f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/9b37900aa35a4e518b68c3d46e964e8f 2024-12-09T19:05:11,967 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/C of 5ae3551814f4d0fc673941d40ec716c2 into 9b37900aa35a4e518b68c3d46e964e8f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:11,968 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:11,968 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/C, priority=13, startTime=1733771111522; duration=0sec 2024-12-09T19:05:11,968 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:11,968 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:C 2024-12-09T19:05:12,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:12,060 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b8805b8ea85d45138d891d9495baee38_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8805b8ea85d45138d891d9495baee38_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:12,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/339167ab204a4fb684242b935705876b, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/339167ab204a4fb684242b935705876b is 175, key is test_row_0/A:col10/1733771109940/Put/seqid=0 2024-12-09T19:05:12,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742157_1333 (size=31105) 2024-12-09T19:05:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:12,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:12,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771172175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:12,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771172284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:12,348 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/b62133903fe646ea8de49bbb8ba80220 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b62133903fe646ea8de49bbb8ba80220 2024-12-09T19:05:12,353 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/A of 5ae3551814f4d0fc673941d40ec716c2 into b62133903fe646ea8de49bbb8ba80220(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:12,353 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:12,353 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/A, priority=13, startTime=1733771111522; duration=0sec 2024-12-09T19:05:12,353 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:12,353 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:A 2024-12-09T19:05:12,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/339167ab204a4fb684242b935705876b 2024-12-09T19:05:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/f4024d38de13438db0765de9a46783cb is 50, key is test_row_0/B:col10/1733771109940/Put/seqid=0 2024-12-09T19:05:12,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742158_1334 (size=12151) 2024-12-09T19:05:12,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771172494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:12,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771172805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:12,888 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/f4024d38de13438db0765de9a46783cb 2024-12-09T19:05:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/96acf14486f545e29815b46bcc3b61f1 is 50, key is test_row_0/C:col10/1733771109940/Put/seqid=0 2024-12-09T19:05:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742159_1335 (size=12151) 2024-12-09T19:05:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771173244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,248 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:13,298 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/96acf14486f545e29815b46bcc3b61f1 2024-12-09T19:05:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/339167ab204a4fb684242b935705876b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b 2024-12-09T19:05:13,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b, entries=150, sequenceid=153, filesize=30.4 K 2024-12-09T19:05:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/f4024d38de13438db0765de9a46783cb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f4024d38de13438db0765de9a46783cb 2024-12-09T19:05:13,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f4024d38de13438db0765de9a46783cb, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T19:05:13,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/96acf14486f545e29815b46bcc3b61f1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/96acf14486f545e29815b46bcc3b61f1 2024-12-09T19:05:13,312 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/96acf14486f545e29815b46bcc3b61f1, entries=150, sequenceid=153, filesize=11.9 K 2024-12-09T19:05:13,312 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5ae3551814f4d0fc673941d40ec716c2 in 1665ms, sequenceid=153, compaction requested=false 2024-12-09T19:05:13,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:13,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:13,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T19:05:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-09T19:05:13,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T19:05:13,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.7870 sec 2024-12-09T19:05:13,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 5.7910 sec 2024-12-09T19:05:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:13,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:13,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-09T19:05:13,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097e6d62c56c0d475e8d0fea9363c94df0_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:13,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742160_1336 (size=14794) 2024-12-09T19:05:13,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771173411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771173524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:13,730 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:13,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097e6d62c56c0d475e8d0fea9363c94df0_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097e6d62c56c0d475e8d0fea9363c94df0_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:13,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/073797df857c4ba684c020c4d4d5a921, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:13,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/073797df857c4ba684c020c4d4d5a921 is 175, key is test_row_0/A:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:13,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742161_1337 (size=39749) 2024-12-09T19:05:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771173734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:13,743 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/073797df857c4ba684c020c4d4d5a921 2024-12-09T19:05:13,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/199db45edd9a49c780e3391b2bbfb469 is 50, key is test_row_0/B:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:13,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742162_1338 (size=12151) 2024-12-09T19:05:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38230 deadline: 1733771173864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,867 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38266 deadline: 1733771173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1733771173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:13,891 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:13,891 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:14,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771174044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:14,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/199db45edd9a49c780e3391b2bbfb469 2024-12-09T19:05:14,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/45f259d79f9c487e9dc8457d971b31f3 is 50, key is test_row_0/C:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742163_1339 (size=12151) 2024-12-09T19:05:14,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771174555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:14,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:14,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/45f259d79f9c487e9dc8457d971b31f3 2024-12-09T19:05:14,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/073797df857c4ba684c020c4d4d5a921 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921 2024-12-09T19:05:14,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921, entries=200, sequenceid=172, filesize=38.8 K 2024-12-09T19:05:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/199db45edd9a49c780e3391b2bbfb469 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/199db45edd9a49c780e3391b2bbfb469 2024-12-09T19:05:14,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/199db45edd9a49c780e3391b2bbfb469, entries=150, sequenceid=172, filesize=11.9 K 2024-12-09T19:05:14,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/45f259d79f9c487e9dc8457d971b31f3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/45f259d79f9c487e9dc8457d971b31f3 2024-12-09T19:05:14,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/45f259d79f9c487e9dc8457d971b31f3, entries=150, sequenceid=172, filesize=11.9 K 2024-12-09T19:05:14,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5ae3551814f4d0fc673941d40ec716c2 in 1268ms, sequenceid=172, compaction requested=true 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:14,588 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:14,588 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:14,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:14,589 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:14,589 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:14,589 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/A is initiating minor compaction (all files) 2024-12-09T19:05:14,589 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/B is initiating minor compaction (all files) 2024-12-09T19:05:14,590 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/A in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:14,590 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/B in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:14,590 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/bdc1479393994d1eab22a3f4331c13b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f4024d38de13438db0765de9a46783cb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/199db45edd9a49c780e3391b2bbfb469] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=35.8 K 2024-12-09T19:05:14,590 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b62133903fe646ea8de49bbb8ba80220, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=99.7 K 2024-12-09T19:05:14,590 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:14,590 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b62133903fe646ea8de49bbb8ba80220, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921] 2024-12-09T19:05:14,590 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting bdc1479393994d1eab22a3f4331c13b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771109035 2024-12-09T19:05:14,590 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b62133903fe646ea8de49bbb8ba80220, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771109035 2024-12-09T19:05:14,590 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f4024d38de13438db0765de9a46783cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733771109940 2024-12-09T19:05:14,590 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339167ab204a4fb684242b935705876b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733771109940 2024-12-09T19:05:14,591 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 073797df857c4ba684c020c4d4d5a921, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733771112160 2024-12-09T19:05:14,591 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 199db45edd9a49c780e3391b2bbfb469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733771112160 2024-12-09T19:05:14,595 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:14,596 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:14,597 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/55d8c5a9111a45e787e00e029736b942 is 50, key is test_row_0/B:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:14,598 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209e018dbdd2bb249f59f066c3dbb044670_5ae3551814f4d0fc673941d40ec716c2 store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:14,599 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209e018dbdd2bb249f59f066c3dbb044670_5ae3551814f4d0fc673941d40ec716c2, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:14,599 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e018dbdd2bb249f59f066c3dbb044670_5ae3551814f4d0fc673941d40ec716c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:14,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742164_1340 (size=12527) 2024-12-09T19:05:14,609 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/55d8c5a9111a45e787e00e029736b942 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/55d8c5a9111a45e787e00e029736b942 2024-12-09T19:05:14,613 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/B of 5ae3551814f4d0fc673941d40ec716c2 into 55d8c5a9111a45e787e00e029736b942(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:14,613 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:14,613 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/B, priority=13, startTime=1733771114588; duration=0sec 2024-12-09T19:05:14,613 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:14,613 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:B 2024-12-09T19:05:14,613 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:14,613 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:14,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/C is initiating minor compaction (all files) 2024-12-09T19:05:14,614 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/C in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:14,614 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/9b37900aa35a4e518b68c3d46e964e8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/96acf14486f545e29815b46bcc3b61f1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/45f259d79f9c487e9dc8457d971b31f3] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=35.8 K 2024-12-09T19:05:14,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b37900aa35a4e518b68c3d46e964e8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771109035 2024-12-09T19:05:14,614 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 96acf14486f545e29815b46bcc3b61f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733771109940 2024-12-09T19:05:14,615 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f259d79f9c487e9dc8457d971b31f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733771112160 2024-12-09T19:05:14,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742165_1341 (size=4469) 2024-12-09T19:05:14,619 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#A#compaction#286 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:14,620 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/ac285f1df23a4c59a929f3a6a4b8fa27 is 175, key is test_row_0/A:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:14,621 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:14,621 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/6acd4aad553840188b283f6aa09f0a85 is 50, key is test_row_0/C:col10/1733771112174/Put/seqid=0 2024-12-09T19:05:14,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742166_1342 (size=31481) 2024-12-09T19:05:14,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742167_1343 (size=12527) 2024-12-09T19:05:15,028 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/ac285f1df23a4c59a929f3a6a4b8fa27 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ac285f1df23a4c59a929f3a6a4b8fa27 2024-12-09T19:05:15,036 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/6acd4aad553840188b283f6aa09f0a85 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/6acd4aad553840188b283f6aa09f0a85 2024-12-09T19:05:15,037 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/A of 5ae3551814f4d0fc673941d40ec716c2 into ac285f1df23a4c59a929f3a6a4b8fa27(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:15,037 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:15,037 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/A, priority=13, startTime=1733771114588; duration=0sec 2024-12-09T19:05:15,037 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:15,037 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:A 2024-12-09T19:05:15,040 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/C of 5ae3551814f4d0fc673941d40ec716c2 into 6acd4aad553840188b283f6aa09f0a85(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:15,040 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:15,040 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/C, priority=13, startTime=1733771114588; duration=0sec 2024-12-09T19:05:15,040 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:15,040 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:C 2024-12-09T19:05:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:15,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:15,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-09T19:05:15,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093ddba6ee3c9a43b4b5a524585f6a93a0_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771113371/Put/seqid=0 2024-12-09T19:05:15,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742168_1344 (size=14794) 2024-12-09T19:05:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771175632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-09T19:05:15,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:15,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T19:05:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:15,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:15,717 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:15,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:15,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771175744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:15,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:15,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:15,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:15,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:15,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:15,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:15,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771175954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:15,986 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:15,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093ddba6ee3c9a43b4b5a524585f6a93a0_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093ddba6ee3c9a43b4b5a524585f6a93a0_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:15,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/4f7afc7814424c928a0d9887eea04056, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:15,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/4f7afc7814424c928a0d9887eea04056 is 175, key is test_row_0/A:col10/1733771113371/Put/seqid=0 2024-12-09T19:05:15,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742169_1345 (size=39749) 2024-12-09T19:05:16,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:16,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:16,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771176265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:16,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:16,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/4f7afc7814424c928a0d9887eea04056 2024-12-09T19:05:16,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/ac0394ce88a14d10acbcc3a4f33663e3 is 50, key is test_row_0/B:col10/1733771113371/Put/seqid=0 2024-12-09T19:05:16,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742170_1346 (size=12151) 2024-12-09T19:05:16,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/ac0394ce88a14d10acbcc3a4f33663e3 2024-12-09T19:05:16,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/606ee5e0c328405fb896ec57b477fed3 is 50, key is test_row_0/C:col10/1733771113371/Put/seqid=0 2024-12-09T19:05:16,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742171_1347 (size=12151) 2024-12-09T19:05:16,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:16,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:16,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,700 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T19:05:16,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771176774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:16,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:16,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:16,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/606ee5e0c328405fb896ec57b477fed3 2024-12-09T19:05:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/4f7afc7814424c928a0d9887eea04056 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056 2024-12-09T19:05:16,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056, entries=200, sequenceid=195, filesize=38.8 K 2024-12-09T19:05:16,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/ac0394ce88a14d10acbcc3a4f33663e3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/ac0394ce88a14d10acbcc3a4f33663e3 2024-12-09T19:05:16,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/ac0394ce88a14d10acbcc3a4f33663e3, entries=150, sequenceid=195, filesize=11.9 K 2024-12-09T19:05:16,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/606ee5e0c328405fb896ec57b477fed3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/606ee5e0c328405fb896ec57b477fed3 2024-12-09T19:05:16,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/606ee5e0c328405fb896ec57b477fed3, entries=150, sequenceid=195, filesize=11.9 K 2024-12-09T19:05:16,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5ae3551814f4d0fc673941d40ec716c2 in 1272ms, sequenceid=195, compaction requested=false 2024-12-09T19:05:16,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:16,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T19:05:16,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:16,940 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-09T19:05:16,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120997b8f907afab4fb99e3faf96c234eb85_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771115631/Put/seqid=0 2024-12-09T19:05:16,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742172_1348 (size=12304) 2024-12-09T19:05:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. as already flushing 2024-12-09T19:05:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771177343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:17,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:17,354 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120997b8f907afab4fb99e3faf96c234eb85_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120997b8f907afab4fb99e3faf96c234eb85_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/438d8b0f17d34bba9ca03d67ac79645d, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/438d8b0f17d34bba9ca03d67ac79645d is 175, key is test_row_0/A:col10/1733771115631/Put/seqid=0 2024-12-09T19:05:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742173_1349 (size=31105) 2024-12-09T19:05:17,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771177455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771177664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:17,759 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/438d8b0f17d34bba9ca03d67ac79645d 2024-12-09T19:05:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/79728c007ca5470e8133aeb87140068f is 50, key is test_row_0/B:col10/1733771115631/Put/seqid=0 2024-12-09T19:05:17,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742174_1350 (size=12151) 2024-12-09T19:05:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1733771177794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:17,798 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in 5ae3551814f4d0fc673941d40ec716c2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771177975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:18,168 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/79728c007ca5470e8133aeb87140068f 2024-12-09T19:05:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/cea30975386b4eda8e0c8c1e3e594929 is 50, key is test_row_0/C:col10/1733771115631/Put/seqid=0 2024-12-09T19:05:18,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742175_1351 (size=12151) 2024-12-09T19:05:18,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38314 deadline: 1733771178484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:18,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/cea30975386b4eda8e0c8c1e3e594929 2024-12-09T19:05:18,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/438d8b0f17d34bba9ca03d67ac79645d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d 2024-12-09T19:05:18,584 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d, entries=150, sequenceid=211, filesize=30.4 K 2024-12-09T19:05:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/79728c007ca5470e8133aeb87140068f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/79728c007ca5470e8133aeb87140068f 2024-12-09T19:05:18,588 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/79728c007ca5470e8133aeb87140068f, entries=150, sequenceid=211, filesize=11.9 K 2024-12-09T19:05:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/cea30975386b4eda8e0c8c1e3e594929 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cea30975386b4eda8e0c8c1e3e594929 2024-12-09T19:05:18,591 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cea30975386b4eda8e0c8c1e3e594929, entries=150, sequenceid=211, filesize=11.9 K 2024-12-09T19:05:18,592 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5ae3551814f4d0fc673941d40ec716c2 in 1652ms, sequenceid=211, compaction requested=true 2024-12-09T19:05:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T19:05:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T19:05:18,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T19:05:18,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8770 sec 2024-12-09T19:05:18,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.8810 sec 2024-12-09T19:05:19,063 INFO [Thread-1386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1386. 2024-12-09T19:05:19,063 DEBUG [Thread-1386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,063 INFO [Thread-1388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1388. 2024-12-09T19:05:19,063 DEBUG [Thread-1386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,063 DEBUG [Thread-1388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,063 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,063 DEBUG [Thread-1386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,063 DEBUG [Thread-1388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,066 INFO [Thread-1384 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1384. 2024-12-09T19:05:19,066 DEBUG [Thread-1384 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,066 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,066 DEBUG [Thread-1384 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,068 INFO [Thread-1382 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1382. 2024-12-09T19:05:19,068 DEBUG [Thread-1382 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,068 DEBUG [Thread-1382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,068 DEBUG [Thread-1382 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,068 INFO [Thread-1390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1390. 2024-12-09T19:05:19,068 DEBUG [Thread-1390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,068 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,068 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:19,509 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:19,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-09T19:05:19,510 INFO [Thread-1373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1373. 2024-12-09T19:05:19,510 DEBUG [Thread-1373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,510 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,510 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a0a394ed4f23474dbaace1979b8e82db_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:19,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742176_1352 (size=12304) 2024-12-09T19:05:19,819 INFO [Thread-1377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1377. 2024-12-09T19:05:19,819 DEBUG [Thread-1377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:19,819 DEBUG [Thread-1377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,820 DEBUG [Thread-1377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:19,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T19:05:19,876 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:19,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:19,935 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a0a394ed4f23474dbaace1979b8e82db_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a0a394ed4f23474dbaace1979b8e82db_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:19,936 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/b5adb7cb80de440faf4906ded265ff8e, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:19,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/b5adb7cb80de440faf4906ded265ff8e is 175, key is test_row_0/A:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:19,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742177_1353 (size=31105) 2024-12-09T19:05:20,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/b5adb7cb80de440faf4906ded265ff8e 2024-12-09T19:05:20,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/6c962ec1628e45a2b6ae95159a3860fa is 50, key is test_row_0/B:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:20,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742178_1354 (size=12151) 2024-12-09T19:05:20,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/6c962ec1628e45a2b6ae95159a3860fa 2024-12-09T19:05:20,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/56a041c5956c4692901bc76d5e189d12 is 50, key is test_row_0/C:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:20,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742179_1355 (size=12151) 2024-12-09T19:05:21,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/752a67358926490caadfdffca496b99f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b62133903fe646ea8de49bbb8ba80220, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921] to archive 2024-12-09T19:05:21,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:21,087 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/9ad9cd6a2d984f41af1bd7f9f8603014 2024-12-09T19:05:21,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/2f76ad73479d49259468b8cd0396c834 2024-12-09T19:05:21,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/6e01e9cdd7c445a48c2d794b230d3352 2024-12-09T19:05:21,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c881cdbc09fe4eb8a8f279ca5903e4ab 2024-12-09T19:05:21,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/057fdf0d300346759f027b4f5f379d7c 2024-12-09T19:05:21,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/752a67358926490caadfdffca496b99f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/752a67358926490caadfdffca496b99f 2024-12-09T19:05:21,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ea60146525cf4b5298f283efc02ab56a 2024-12-09T19:05:21,094 WARN [regionserver/a437f8b9ba7d:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-09T19:05:21,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b62133903fe646ea8de49bbb8ba80220 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b62133903fe646ea8de49bbb8ba80220 2024-12-09T19:05:21,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/816d32df6771438884219f381526e44b 2024-12-09T19:05:21,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/339167ab204a4fb684242b935705876b 2024-12-09T19:05:21,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/073797df857c4ba684c020c4d4d5a921 2024-12-09T19:05:21,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b3aabd76dcb4419b98892ae46ca29935, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b97e2ba5282945bdbd5995351384fdd4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/565a6090b79a49f488e2adb74e61f059, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/d6516ee3644a4c93aad0e83c32c35ae4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/76ca2d90f81940e090005a1fb17c3e74, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/639f953377034ce5a1577cd376d312e7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/0243d6f623024692ad6b2863320b94c9, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/bdc1479393994d1eab22a3f4331c13b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/e493c538652d42e0a88a25b3b68d7629, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f4024d38de13438db0765de9a46783cb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/199db45edd9a49c780e3391b2bbfb469] to archive 2024-12-09T19:05:21,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:21,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b3aabd76dcb4419b98892ae46ca29935 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b3aabd76dcb4419b98892ae46ca29935 2024-12-09T19:05:21,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b97e2ba5282945bdbd5995351384fdd4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/b97e2ba5282945bdbd5995351384fdd4 2024-12-09T19:05:21,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/565a6090b79a49f488e2adb74e61f059 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/565a6090b79a49f488e2adb74e61f059 2024-12-09T19:05:21,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/d6516ee3644a4c93aad0e83c32c35ae4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/d6516ee3644a4c93aad0e83c32c35ae4 2024-12-09T19:05:21,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/76ca2d90f81940e090005a1fb17c3e74 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/76ca2d90f81940e090005a1fb17c3e74 2024-12-09T19:05:21,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/639f953377034ce5a1577cd376d312e7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/639f953377034ce5a1577cd376d312e7 2024-12-09T19:05:21,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/0243d6f623024692ad6b2863320b94c9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/0243d6f623024692ad6b2863320b94c9 2024-12-09T19:05:21,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/bdc1479393994d1eab22a3f4331c13b4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/bdc1479393994d1eab22a3f4331c13b4 2024-12-09T19:05:21,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/e493c538652d42e0a88a25b3b68d7629 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/e493c538652d42e0a88a25b3b68d7629 2024-12-09T19:05:21,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f4024d38de13438db0765de9a46783cb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f4024d38de13438db0765de9a46783cb 2024-12-09T19:05:21,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/199db45edd9a49c780e3391b2bbfb469 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/199db45edd9a49c780e3391b2bbfb469 2024-12-09T19:05:21,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/c78558ff5583474abc5ca7d7f633f438, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/fb3c53bbd23f45eca69a3a665beb9d84, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/19215e0e369a4fc887091bc3010fc07f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3406cea5f28844789ef5ae597f5af791, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3b50ffe4f576486cb91e8d1f8595dff0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/8e83110a24e4473081f1605c429ee008, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/77a4ffb428a1429989ec9468a63789d6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/9b37900aa35a4e518b68c3d46e964e8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/2c5cfa8608f24b13833890f5887efbab, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/96acf14486f545e29815b46bcc3b61f1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/45f259d79f9c487e9dc8457d971b31f3] to archive 2024-12-09T19:05:21,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:21,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/c78558ff5583474abc5ca7d7f633f438 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/c78558ff5583474abc5ca7d7f633f438 2024-12-09T19:05:21,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/fb3c53bbd23f45eca69a3a665beb9d84 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/fb3c53bbd23f45eca69a3a665beb9d84 2024-12-09T19:05:21,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/19215e0e369a4fc887091bc3010fc07f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/19215e0e369a4fc887091bc3010fc07f 2024-12-09T19:05:21,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3406cea5f28844789ef5ae597f5af791 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3406cea5f28844789ef5ae597f5af791 2024-12-09T19:05:21,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3b50ffe4f576486cb91e8d1f8595dff0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/3b50ffe4f576486cb91e8d1f8595dff0 2024-12-09T19:05:21,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/8e83110a24e4473081f1605c429ee008 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/8e83110a24e4473081f1605c429ee008 2024-12-09T19:05:21,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/77a4ffb428a1429989ec9468a63789d6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/77a4ffb428a1429989ec9468a63789d6 2024-12-09T19:05:21,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/9b37900aa35a4e518b68c3d46e964e8f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/9b37900aa35a4e518b68c3d46e964e8f 2024-12-09T19:05:21,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/2c5cfa8608f24b13833890f5887efbab to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/2c5cfa8608f24b13833890f5887efbab 2024-12-09T19:05:21,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/96acf14486f545e29815b46bcc3b61f1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/96acf14486f545e29815b46bcc3b61f1 2024-12-09T19:05:21,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/45f259d79f9c487e9dc8457d971b31f3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/45f259d79f9c487e9dc8457d971b31f3 2024-12-09T19:05:21,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/56a041c5956c4692901bc76d5e189d12 2024-12-09T19:05:21,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/b5adb7cb80de440faf4906ded265ff8e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e 2024-12-09T19:05:21,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e, entries=150, sequenceid=232, filesize=30.4 K 2024-12-09T19:05:21,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/6c962ec1628e45a2b6ae95159a3860fa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/6c962ec1628e45a2b6ae95159a3860fa 2024-12-09T19:05:21,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/6c962ec1628e45a2b6ae95159a3860fa, entries=150, sequenceid=232, filesize=11.9 K 2024-12-09T19:05:21,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/56a041c5956c4692901bc76d5e189d12 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/56a041c5956c4692901bc76d5e189d12 2024-12-09T19:05:21,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/56a041c5956c4692901bc76d5e189d12, entries=150, sequenceid=232, filesize=11.9 K 2024-12-09T19:05:21,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=6.71 KB/6870 for 5ae3551814f4d0fc673941d40ec716c2 in 1690ms, sequenceid=232, compaction requested=true 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:21,199 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ae3551814f4d0fc673941d40ec716c2:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:21,199 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:21,200 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:21,200 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:21,200 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/B is initiating minor compaction (all files) 2024-12-09T19:05:21,200 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/A is initiating minor compaction (all files) 2024-12-09T19:05:21,200 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/B in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:21,200 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/A in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:21,200 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/55d8c5a9111a45e787e00e029736b942, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/ac0394ce88a14d10acbcc3a4f33663e3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/79728c007ca5470e8133aeb87140068f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/6c962ec1628e45a2b6ae95159a3860fa] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=47.8 K 2024-12-09T19:05:21,200 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ac285f1df23a4c59a929f3a6a4b8fa27, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=130.3 K 2024-12-09T19:05:21,201 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ac285f1df23a4c59a929f3a6a4b8fa27, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e] 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 55d8c5a9111a45e787e00e029736b942, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733771112160 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac285f1df23a4c59a929f3a6a4b8fa27, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733771112160 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0394ce88a14d10acbcc3a4f33663e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771113371 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f7afc7814424c928a0d9887eea04056, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771113371 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 79728c007ca5470e8133aeb87140068f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733771115590 2024-12-09T19:05:21,201 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438d8b0f17d34bba9ca03d67ac79645d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733771115590 2024-12-09T19:05:21,202 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c962ec1628e45a2b6ae95159a3860fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733771117333 2024-12-09T19:05:21,202 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5adb7cb80de440faf4906ded265ff8e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733771117333 2024-12-09T19:05:21,208 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:21,209 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/f0e70d3ece864c6285564f4bb5c16bce is 50, key is test_row_0/B:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:21,211 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:21,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742180_1356 (size=12289) 2024-12-09T19:05:21,212 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412091a53cd6601d942d69708deb678cd77eb_5ae3551814f4d0fc673941d40ec716c2 store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:21,230 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412091a53cd6601d942d69708deb678cd77eb_5ae3551814f4d0fc673941d40ec716c2, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:21,230 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091a53cd6601d942d69708deb678cd77eb_5ae3551814f4d0fc673941d40ec716c2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742181_1357 (size=4469) 2024-12-09T19:05:21,237 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#A#compaction#298 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:21,238 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/c53dd7a6a971465292f5463a2631988a is 175, key is test_row_0/A:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:21,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742182_1358 (size=31243) 2024-12-09T19:05:21,646 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/f0e70d3ece864c6285564f4bb5c16bce as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f0e70d3ece864c6285564f4bb5c16bce 2024-12-09T19:05:21,650 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/c53dd7a6a971465292f5463a2631988a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c53dd7a6a971465292f5463a2631988a 2024-12-09T19:05:21,653 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/B of 5ae3551814f4d0fc673941d40ec716c2 into f0e70d3ece864c6285564f4bb5c16bce(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:21,654 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:21,654 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/B, priority=12, startTime=1733771121199; duration=0sec 2024-12-09T19:05:21,654 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:21,654 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:B 2024-12-09T19:05:21,654 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:21,656 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:21,656 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): 5ae3551814f4d0fc673941d40ec716c2/C is initiating minor compaction (all files) 2024-12-09T19:05:21,656 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ae3551814f4d0fc673941d40ec716c2/C in TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:21,656 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/6acd4aad553840188b283f6aa09f0a85, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/606ee5e0c328405fb896ec57b477fed3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cea30975386b4eda8e0c8c1e3e594929, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/56a041c5956c4692901bc76d5e189d12] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp, totalSize=47.8 K 2024-12-09T19:05:21,656 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 6acd4aad553840188b283f6aa09f0a85, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733771112160 2024-12-09T19:05:21,656 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/A of 5ae3551814f4d0fc673941d40ec716c2 into c53dd7a6a971465292f5463a2631988a(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:21,657 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:21,657 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/A, priority=12, startTime=1733771121199; duration=0sec 2024-12-09T19:05:21,657 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:21,657 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:A 2024-12-09T19:05:21,657 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 606ee5e0c328405fb896ec57b477fed3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733771113371 2024-12-09T19:05:21,657 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting cea30975386b4eda8e0c8c1e3e594929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733771115590 2024-12-09T19:05:21,658 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a041c5956c4692901bc76d5e189d12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733771117333 2024-12-09T19:05:21,668 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ae3551814f4d0fc673941d40ec716c2#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:21,669 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/ae53a0dfdc89433695af1f001d30df46 is 50, key is test_row_0/C:col10/1733771117339/Put/seqid=0 2024-12-09T19:05:21,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742183_1359 (size=12289) 2024-12-09T19:05:22,087 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/ae53a0dfdc89433695af1f001d30df46 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/ae53a0dfdc89433695af1f001d30df46 2024-12-09T19:05:22,091 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5ae3551814f4d0fc673941d40ec716c2/C of 5ae3551814f4d0fc673941d40ec716c2 into ae53a0dfdc89433695af1f001d30df46(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:22,091 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ae3551814f4d0fc673941d40ec716c2: 2024-12-09T19:05:22,091 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., storeName=5ae3551814f4d0fc673941d40ec716c2/C, priority=12, startTime=1733771121199; duration=0sec 2024-12-09T19:05:22,091 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:22,091 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ae3551814f4d0fc673941d40ec716c2:C 2024-12-09T19:05:23,919 INFO [Thread-1375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1375. 2024-12-09T19:05:23,919 DEBUG [Thread-1375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:23,919 INFO [Thread-1379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1379. 2024-12-09T19:05:23,919 DEBUG [Thread-1375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,920 DEBUG [Thread-1379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:23,920 DEBUG [Thread-1379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,920 DEBUG [Thread-1375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:23,920 DEBUG [Thread-1379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:23,927 INFO [Thread-1371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1371. 2024-12-09T19:05:23,927 DEBUG [Thread-1371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:23,927 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,927 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:23,927 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1903 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5707 rows 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1892 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5671 rows 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1900 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5698 rows 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1904 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5710 rows 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1904 2024-12-09T19:05:23,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5712 rows 2024-12-09T19:05:23,928 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:05:23,929 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:05:23,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:23,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:23,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T19:05:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T19:05:23,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771123933"}]},"ts":"1733771123933"} 2024-12-09T19:05:23,935 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T19:05:23,935 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T19:05:23,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:05:23,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, UNASSIGN}] 2024-12-09T19:05:23,937 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, UNASSIGN 2024-12-09T19:05:23,938 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:23,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, UNASSIGN because future has completed 2024-12-09T19:05:23,939 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:05:23,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:05:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T19:05:24,092 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:24,092 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:05:24,092 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 5ae3551814f4d0fc673941d40ec716c2, disabling compactions & flushes 2024-12-09T19:05:24,092 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:24,093 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:24,093 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. after waiting 0 ms 2024-12-09T19:05:24,093 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:24,093 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 5ae3551814f4d0fc673941d40ec716c2 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-09T19:05:24,105 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209018c18a5d1de413f8bbc92704115d0bd_5ae3551814f4d0fc673941d40ec716c2 is 50, key is test_row_0/A:col10/1733771123916/Put/seqid=0 2024-12-09T19:05:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742184_1360 (size=9814) 2024-12-09T19:05:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T19:05:24,511 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:24,520 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209018c18a5d1de413f8bbc92704115d0bd_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209018c18a5d1de413f8bbc92704115d0bd_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:24,522 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/179a24c8de424db289522a511e644aa9, store: [table=TestAcidGuarantees family=A region=5ae3551814f4d0fc673941d40ec716c2] 2024-12-09T19:05:24,523 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/179a24c8de424db289522a511e644aa9 is 175, key is test_row_0/A:col10/1733771123916/Put/seqid=0 2024-12-09T19:05:24,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742185_1361 (size=22461) 2024-12-09T19:05:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T19:05:24,929 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/179a24c8de424db289522a511e644aa9 2024-12-09T19:05:24,943 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/a12c3fff30aa4426b183fd395dd06944 is 50, key is test_row_0/B:col10/1733771123916/Put/seqid=0 2024-12-09T19:05:24,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742186_1362 (size=9757) 2024-12-09T19:05:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T19:05:25,348 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/a12c3fff30aa4426b183fd395dd06944 2024-12-09T19:05:25,360 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/cf019bb6a84c4b82bf4a1cf5bc6e202d is 50, key is test_row_0/C:col10/1733771123916/Put/seqid=0 2024-12-09T19:05:25,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742187_1363 (size=9757) 2024-12-09T19:05:25,765 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/cf019bb6a84c4b82bf4a1cf5bc6e202d 2024-12-09T19:05:25,769 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/A/179a24c8de424db289522a511e644aa9 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/179a24c8de424db289522a511e644aa9 2024-12-09T19:05:25,772 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/179a24c8de424db289522a511e644aa9, entries=100, sequenceid=242, filesize=21.9 K 2024-12-09T19:05:25,772 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/B/a12c3fff30aa4426b183fd395dd06944 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/a12c3fff30aa4426b183fd395dd06944 2024-12-09T19:05:25,776 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/a12c3fff30aa4426b183fd395dd06944, entries=100, sequenceid=242, filesize=9.5 K 2024-12-09T19:05:25,776 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/.tmp/C/cf019bb6a84c4b82bf4a1cf5bc6e202d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cf019bb6a84c4b82bf4a1cf5bc6e202d 2024-12-09T19:05:25,780 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cf019bb6a84c4b82bf4a1cf5bc6e202d, entries=100, sequenceid=242, filesize=9.5 K 2024-12-09T19:05:25,781 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5ae3551814f4d0fc673941d40ec716c2 in 1688ms, sequenceid=242, compaction requested=false 2024-12-09T19:05:25,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ac285f1df23a4c59a929f3a6a4b8fa27, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e] to archive 2024-12-09T19:05:25,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:25,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ac285f1df23a4c59a929f3a6a4b8fa27 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/ac285f1df23a4c59a929f3a6a4b8fa27 2024-12-09T19:05:25,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/4f7afc7814424c928a0d9887eea04056 2024-12-09T19:05:25,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/438d8b0f17d34bba9ca03d67ac79645d 2024-12-09T19:05:25,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/b5adb7cb80de440faf4906ded265ff8e 2024-12-09T19:05:25,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/55d8c5a9111a45e787e00e029736b942, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/ac0394ce88a14d10acbcc3a4f33663e3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/79728c007ca5470e8133aeb87140068f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/6c962ec1628e45a2b6ae95159a3860fa] to archive 2024-12-09T19:05:25,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:25,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/55d8c5a9111a45e787e00e029736b942 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/55d8c5a9111a45e787e00e029736b942 2024-12-09T19:05:25,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/ac0394ce88a14d10acbcc3a4f33663e3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/ac0394ce88a14d10acbcc3a4f33663e3 2024-12-09T19:05:25,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/79728c007ca5470e8133aeb87140068f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/79728c007ca5470e8133aeb87140068f 2024-12-09T19:05:25,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/6c962ec1628e45a2b6ae95159a3860fa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/6c962ec1628e45a2b6ae95159a3860fa 2024-12-09T19:05:25,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/6acd4aad553840188b283f6aa09f0a85, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/606ee5e0c328405fb896ec57b477fed3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cea30975386b4eda8e0c8c1e3e594929, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/56a041c5956c4692901bc76d5e189d12] to archive 2024-12-09T19:05:25,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:25,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/6acd4aad553840188b283f6aa09f0a85 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/6acd4aad553840188b283f6aa09f0a85 2024-12-09T19:05:25,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/606ee5e0c328405fb896ec57b477fed3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/606ee5e0c328405fb896ec57b477fed3 2024-12-09T19:05:25,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cea30975386b4eda8e0c8c1e3e594929 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cea30975386b4eda8e0c8c1e3e594929 2024-12-09T19:05:25,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/56a041c5956c4692901bc76d5e189d12 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/56a041c5956c4692901bc76d5e189d12 2024-12-09T19:05:25,800 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/recovered.edits/245.seqid, newMaxSeqId=245, maxSeqId=4 2024-12-09T19:05:25,801 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. 2024-12-09T19:05:25,801 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 5ae3551814f4d0fc673941d40ec716c2: Waiting for close lock at 1733771124092Running coprocessor pre-close hooks at 1733771124092Disabling compacts and flushes for region at 1733771124092Disabling writes for close at 1733771124093 (+1 ms)Obtaining lock to block concurrent updates at 1733771124093Preparing flush snapshotting stores in 5ae3551814f4d0fc673941d40ec716c2 at 1733771124093Finished memstore snapshotting TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733771124094 (+1 ms)Flushing stores of TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2. at 1733771124095 (+1 ms)Flushing 5ae3551814f4d0fc673941d40ec716c2/A: creating writer at 1733771124096 (+1 ms)Flushing mob file 5ae3551814f4d0fc673941d40ec716c2/A: appending metadata at 1733771124105 (+9 ms)Flushing mob file 5ae3551814f4d0fc673941d40ec716c2/A: closing flushed file at 1733771124105Flushing 5ae3551814f4d0fc673941d40ec716c2/A: appending metadata at 1733771124522 (+417 ms)Flushing 5ae3551814f4d0fc673941d40ec716c2/A: closing flushed file at 1733771124522Flushing 5ae3551814f4d0fc673941d40ec716c2/B: creating writer at 1733771124938 (+416 ms)Flushing 5ae3551814f4d0fc673941d40ec716c2/B: appending metadata at 1733771124943 (+5 ms)Flushing 5ae3551814f4d0fc673941d40ec716c2/B: closing flushed file at 1733771124943Flushing 5ae3551814f4d0fc673941d40ec716c2/C: creating writer at 1733771125357 (+414 ms)Flushing 5ae3551814f4d0fc673941d40ec716c2/C: appending metadata at 1733771125360 (+3 ms)Flushing 5ae3551814f4d0fc673941d40ec716c2/C: closing flushed file at 1733771125360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2298e3: reopening flushed file at 1733771125768 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52688946: reopening flushed file at 1733771125772 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fb72ff6: reopening flushed file at 1733771125776 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5ae3551814f4d0fc673941d40ec716c2 in 1688ms, sequenceid=242, compaction requested=false at 1733771125781 (+5 ms)Writing region close event to WAL at 1733771125798 (+17 ms)Running coprocessor post-close hooks at 1733771125800 (+2 ms)Closed at 1733771125800 2024-12-09T19:05:25,802 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:25,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=5ae3551814f4d0fc673941d40ec716c2, regionState=CLOSED 2024-12-09T19:05:25,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:05:25,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T19:05:25,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 5ae3551814f4d0fc673941d40ec716c2, server=a437f8b9ba7d,46189,1733770998237 in 1.8650 sec 2024-12-09T19:05:25,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-12-09T19:05:25,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5ae3551814f4d0fc673941d40ec716c2, UNASSIGN in 1.8690 sec 2024-12-09T19:05:25,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T19:05:25,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8720 sec 2024-12-09T19:05:25,809 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771125809"}]},"ts":"1733771125809"} 2024-12-09T19:05:25,810 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T19:05:25,810 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T19:05:25,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8800 sec 2024-12-09T19:05:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T19:05:26,076 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:26,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T19:05:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,082 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T19:05:26,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,087 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,090 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/recovered.edits] 2024-12-09T19:05:26,094 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/179a24c8de424db289522a511e644aa9 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/179a24c8de424db289522a511e644aa9 2024-12-09T19:05:26,096 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c53dd7a6a971465292f5463a2631988a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/A/c53dd7a6a971465292f5463a2631988a 2024-12-09T19:05:26,098 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/a12c3fff30aa4426b183fd395dd06944 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/a12c3fff30aa4426b183fd395dd06944 2024-12-09T19:05:26,100 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f0e70d3ece864c6285564f4bb5c16bce to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/B/f0e70d3ece864c6285564f4bb5c16bce 2024-12-09T19:05:26,104 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/ae53a0dfdc89433695af1f001d30df46 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/ae53a0dfdc89433695af1f001d30df46 2024-12-09T19:05:26,106 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cf019bb6a84c4b82bf4a1cf5bc6e202d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/C/cf019bb6a84c4b82bf4a1cf5bc6e202d 2024-12-09T19:05:26,109 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/recovered.edits/245.seqid to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2/recovered.edits/245.seqid 2024-12-09T19:05:26,110 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,110 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T19:05:26,111 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T19:05:26,112 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T19:05:26,115 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209018c18a5d1de413f8bbc92704115d0bd_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209018c18a5d1de413f8bbc92704115d0bd_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,117 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209024d6b5d72ba48159e2b2faaa67a9754_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209024d6b5d72ba48159e2b2faaa67a9754_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,119 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120914bb32ebab4e47a09dfe57697940e4e0_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120914bb32ebab4e47a09dfe57697940e4e0_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,121 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091c3abda85bcc4fb8a18352b11d235245_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091c3abda85bcc4fb8a18352b11d235245_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,123 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093365071752fe4f3d8fe776a96bb0b68f_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093365071752fe4f3d8fe776a96bb0b68f_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,124 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093ddba6ee3c9a43b4b5a524585f6a93a0_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093ddba6ee3c9a43b4b5a524585f6a93a0_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,126 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097e6d62c56c0d475e8d0fea9363c94df0_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097e6d62c56c0d475e8d0fea9363c94df0_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,127 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098483104113684beebb7bcdea05374af9_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098483104113684beebb7bcdea05374af9_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,127 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209850876e211b44c208c46dca882b2b63c_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209850876e211b44c208c46dca882b2b63c_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,128 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098e927a79e2054e858d61228d84d211e7_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098e927a79e2054e858d61228d84d211e7_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,129 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120997b8f907afab4fb99e3faf96c234eb85_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120997b8f907afab4fb99e3faf96c234eb85_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,129 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a0a394ed4f23474dbaace1979b8e82db_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a0a394ed4f23474dbaace1979b8e82db_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8805b8ea85d45138d891d9495baee38_5ae3551814f4d0fc673941d40ec716c2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b8805b8ea85d45138d891d9495baee38_5ae3551814f4d0fc673941d40ec716c2 2024-12-09T19:05:26,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T19:05:26,132 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,133 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T19:05:26,134 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T19:05:26,135 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,135 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T19:05:26,135 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733771126135"}]},"ts":"9223372036854775807"} 2024-12-09T19:05:26,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T19:05:26,136 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5ae3551814f4d0fc673941d40ec716c2, NAME => 'TestAcidGuarantees,,1733771095887.5ae3551814f4d0fc673941d40ec716c2.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T19:05:26,136 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T19:05:26,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733771126136"}]},"ts":"9223372036854775807"} 2024-12-09T19:05:26,138 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T19:05:26,138 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-12-09T19:05:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T19:05:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T19:05:26,195 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:26,209 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239 (was 238) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 282) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2169 (was 2202) 2024-12-09T19:05:26,219 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=2169 2024-12-09T19:05:26,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:05:26,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:05:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:26,224 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T19:05:26,224 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:26,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-12-09T19:05:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-09T19:05:26,225 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T19:05:26,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742188_1364 (size=959) 2024-12-09T19:05:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-09T19:05:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-09T19:05:26,637 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:05:26,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742189_1365 (size=53) 2024-12-09T19:05:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-09T19:05:27,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:05:27,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e7537c2a1583e00ca6f87dcf9620724c, disabling compactions & flushes 2024-12-09T19:05:27,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. after waiting 0 ms 2024-12-09T19:05:27,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7537c2a1583e00ca6f87dcf9620724c: Waiting for close lock at 1733771127085Disabling compacts and flushes for region at 1733771127085Disabling writes for close at 1733771127085Writing region close event to WAL at 1733771127085Closed at 1733771127085 2024-12-09T19:05:27,087 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T19:05:27,087 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733771127087"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733771127087"}]},"ts":"1733771127087"} 2024-12-09T19:05:27,090 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T19:05:27,091 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T19:05:27,091 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771127091"}]},"ts":"1733771127091"} 2024-12-09T19:05:27,093 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T19:05:27,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, ASSIGN}] 2024-12-09T19:05:27,095 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, ASSIGN 2024-12-09T19:05:27,096 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=false 2024-12-09T19:05:27,247 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e7537c2a1583e00ca6f87dcf9620724c, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:27,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, ASSIGN because future has completed 2024-12-09T19:05:27,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:05:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-09T19:05:27,414 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,415 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:05:27,415 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,416 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:05:27,416 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,416 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,417 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,418 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7537c2a1583e00ca6f87dcf9620724c columnFamilyName A 2024-12-09T19:05:27,418 DEBUG [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:27,419 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] regionserver.HStore(327): Store=e7537c2a1583e00ca6f87dcf9620724c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:27,419 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,420 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7537c2a1583e00ca6f87dcf9620724c columnFamilyName B 2024-12-09T19:05:27,420 DEBUG [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:27,420 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] regionserver.HStore(327): Store=e7537c2a1583e00ca6f87dcf9620724c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:27,420 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,421 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7537c2a1583e00ca6f87dcf9620724c columnFamilyName C 2024-12-09T19:05:27,421 DEBUG [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:27,422 INFO [StoreOpener-e7537c2a1583e00ca6f87dcf9620724c-1 {}] regionserver.HStore(327): Store=e7537c2a1583e00ca6f87dcf9620724c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:27,422 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,423 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,423 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,425 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,425 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,425 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:05:27,426 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,428 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:05:27,429 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened e7537c2a1583e00ca6f87dcf9620724c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63575888, jitterRate=-0.05264544486999512}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:05:27,429 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:27,429 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for e7537c2a1583e00ca6f87dcf9620724c: Running coprocessor pre-open hook at 1733771127416Writing region info on filesystem at 1733771127416Initializing all the Stores at 1733771127416Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771127417 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771127417Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771127417Cleaning up temporary data from old regions at 1733771127425 (+8 ms)Running coprocessor post-open hooks at 1733771127429 (+4 ms)Region opened successfully at 1733771127429 2024-12-09T19:05:27,430 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., pid=108, masterSystemTime=1733771127406 2024-12-09T19:05:27,432 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,432 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:27,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=e7537c2a1583e00ca6f87dcf9620724c, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:27,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:05:27,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T19:05:27,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 in 183 msec 2024-12-09T19:05:27,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-09T19:05:27,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, ASSIGN in 343 msec 2024-12-09T19:05:27,439 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T19:05:27,439 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771127439"}]},"ts":"1733771127439"} 2024-12-09T19:05:27,440 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T19:05:27,441 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T19:05:27,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2200 sec 2024-12-09T19:05:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-09T19:05:28,386 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:28,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33512826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,393 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e8fe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,394 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,397 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:05:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:05:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@688bcc0d 2024-12-09T19:05:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:05:28,399 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:05:28,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b31614a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3297f36f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21523032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416837a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,409 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53653903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,411 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69326052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,413 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349227dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7096d669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce0c628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd22741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bec3f7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e200003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,425 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b58318f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a420dbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,429 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464a34da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260ffdae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ef2d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c525056, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:28,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:28,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e3c6d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:28,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55404554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:28,438 DEBUG [Thread-1678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,438 DEBUG [Thread-1693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,439 DEBUG [Thread-1682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,439 DEBUG [Thread-1689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,440 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-09T19:05:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,440 DEBUG [Thread-1695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,440 DEBUG [Thread-1686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,440 DEBUG [Thread-1691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T19:05:28,440 DEBUG [Thread-1684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,441 DEBUG [Thread-1695 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,441 DEBUG [Thread-1697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:28,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,441 DEBUG [Thread-1695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,441 DEBUG [Thread-1691 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,442 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:28,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,443 DEBUG [Thread-1684 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:28,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2] 2024-12-09T19:05:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:28,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:05:28,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/5b9930c9b7494ddd9446b1fa54a1e69c is 50, key is test_row_0/A:col10/1733771128445/Put/seqid=0 2024-12-09T19:05:28,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742190_1366 (size=19021) 2024-12-09T19:05:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771188481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771188480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771188486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771188488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771188488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T19:05:28,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771188595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771188595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771188595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,598 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T19:05:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:28,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771188604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771188604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T19:05:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:28,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T19:05:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771188804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771188804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771188804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771188815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771188815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:28,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/5b9930c9b7494ddd9446b1fa54a1e69c 2024-12-09T19:05:28,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T19:05:28,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:28,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:28,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/4fed995bcd9f4818b457c146c5ce7711 is 50, key is test_row_0/B:col10/1733771128445/Put/seqid=0 2024-12-09T19:05:28,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742191_1367 (size=12001) 2024-12-09T19:05:28,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/4fed995bcd9f4818b457c146c5ce7711 2024-12-09T19:05:28,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/94b32b1cf17a44fbbbcdb46c3a5d1237 is 50, key is test_row_0/C:col10/1733771128445/Put/seqid=0 2024-12-09T19:05:28,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742192_1368 (size=12001) 2024-12-09T19:05:29,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T19:05:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:29,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T19:05:29,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771189114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771189114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771189114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771189124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771189125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T19:05:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:29,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:29,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/94b32b1cf17a44fbbbcdb46c3a5d1237 2024-12-09T19:05:29,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/5b9930c9b7494ddd9446b1fa54a1e69c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/5b9930c9b7494ddd9446b1fa54a1e69c 2024-12-09T19:05:29,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/5b9930c9b7494ddd9446b1fa54a1e69c, entries=300, sequenceid=13, filesize=18.6 K 2024-12-09T19:05:29,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/4fed995bcd9f4818b457c146c5ce7711 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/4fed995bcd9f4818b457c146c5ce7711 2024-12-09T19:05:29,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/4fed995bcd9f4818b457c146c5ce7711, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T19:05:29,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/94b32b1cf17a44fbbbcdb46c3a5d1237 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/94b32b1cf17a44fbbbcdb46c3a5d1237 2024-12-09T19:05:29,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/94b32b1cf17a44fbbbcdb46c3a5d1237, entries=150, sequenceid=13, filesize=11.7 K 2024-12-09T19:05:29,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e7537c2a1583e00ca6f87dcf9620724c in 902ms, sequenceid=13, compaction requested=false 2024-12-09T19:05:29,355 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T19:05:29,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:29,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T19:05:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:29,364 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:05:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/23c8f9dc324d4ee2a17cf82381001b50 is 50, key is test_row_0/A:col10/1733771128485/Put/seqid=0 2024-12-09T19:05:29,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742193_1369 (size=12001) 2024-12-09T19:05:29,372 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/23c8f9dc324d4ee2a17cf82381001b50 2024-12-09T19:05:29,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5 is 50, key is test_row_0/B:col10/1733771128485/Put/seqid=0 2024-12-09T19:05:29,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742194_1370 (size=12001) 2024-12-09T19:05:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T19:05:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:29,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:29,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771189634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771189635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771189635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771189636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771189637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771189744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771189744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771189744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771189744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771189754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,783 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5 2024-12-09T19:05:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/8b0bad5f6c394ae7addf898bd2e82e2c is 50, key is test_row_0/C:col10/1733771128485/Put/seqid=0 2024-12-09T19:05:29,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742195_1371 (size=12001) 2024-12-09T19:05:29,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771189954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771189954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771189954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771189954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:29,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771189964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,193 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/8b0bad5f6c394ae7addf898bd2e82e2c 2024-12-09T19:05:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/23c8f9dc324d4ee2a17cf82381001b50 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/23c8f9dc324d4ee2a17cf82381001b50 2024-12-09T19:05:30,200 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/23c8f9dc324d4ee2a17cf82381001b50, entries=150, sequenceid=38, filesize=11.7 K 2024-12-09T19:05:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5 2024-12-09T19:05:30,204 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5, entries=150, sequenceid=38, filesize=11.7 K 2024-12-09T19:05:30,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/8b0bad5f6c394ae7addf898bd2e82e2c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/8b0bad5f6c394ae7addf898bd2e82e2c 2024-12-09T19:05:30,208 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/8b0bad5f6c394ae7addf898bd2e82e2c, entries=150, sequenceid=38, filesize=11.7 K 2024-12-09T19:05:30,209 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e7537c2a1583e00ca6f87dcf9620724c in 845ms, sequenceid=38, compaction requested=false 2024-12-09T19:05:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-09T19:05:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-09T19:05:30,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T19:05:30,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7700 sec 2024-12-09T19:05:30,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7760 sec 2024-12-09T19:05:30,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:30,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:05:30,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/289e5befef414081bb73ed2cd65e0e77 is 50, key is test_row_0/A:col10/1733771129634/Put/seqid=0 2024-12-09T19:05:30,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742196_1372 (size=16681) 2024-12-09T19:05:30,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/289e5befef414081bb73ed2cd65e0e77 2024-12-09T19:05:30,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/2bd2c6fd145546cdb79e72fdc93a2f54 is 50, key is test_row_0/B:col10/1733771129634/Put/seqid=0 2024-12-09T19:05:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742197_1373 (size=12001) 2024-12-09T19:05:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771190297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771190297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771190298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771190299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771190300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771190414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771190414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771190414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771190415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771190415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T19:05:30,584 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:30,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-09T19:05:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T19:05:30,586 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:30,587 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:30,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771190624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771190624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771190625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771190625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771190625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/2bd2c6fd145546cdb79e72fdc93a2f54 2024-12-09T19:05:30,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/be5eda9fde054c72b22f7bb6a892bd48 is 50, key is test_row_0/C:col10/1733771129634/Put/seqid=0 2024-12-09T19:05:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T19:05:30,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742198_1374 (size=12001) 2024-12-09T19:05:30,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/be5eda9fde054c72b22f7bb6a892bd48 2024-12-09T19:05:30,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/289e5befef414081bb73ed2cd65e0e77 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/289e5befef414081bb73ed2cd65e0e77 2024-12-09T19:05:30,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/289e5befef414081bb73ed2cd65e0e77, entries=250, sequenceid=51, filesize=16.3 K 2024-12-09T19:05:30,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/2bd2c6fd145546cdb79e72fdc93a2f54 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/2bd2c6fd145546cdb79e72fdc93a2f54 2024-12-09T19:05:30,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/2bd2c6fd145546cdb79e72fdc93a2f54, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T19:05:30,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/be5eda9fde054c72b22f7bb6a892bd48 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/be5eda9fde054c72b22f7bb6a892bd48 2024-12-09T19:05:30,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/be5eda9fde054c72b22f7bb6a892bd48, entries=150, sequenceid=51, filesize=11.7 K 2024-12-09T19:05:30,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e7537c2a1583e00ca6f87dcf9620724c in 448ms, sequenceid=51, compaction requested=true 2024-12-09T19:05:30,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:30,717 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:30,717 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:30,718 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:30,718 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:30,718 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/5b9930c9b7494ddd9446b1fa54a1e69c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/23c8f9dc324d4ee2a17cf82381001b50, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/289e5befef414081bb73ed2cd65e0e77] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=46.6 K 2024-12-09T19:05:30,718 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/4fed995bcd9f4818b457c146c5ce7711, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/2bd2c6fd145546cdb79e72fdc93a2f54] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=35.2 K 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9930c9b7494ddd9446b1fa54a1e69c, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733771128445 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fed995bcd9f4818b457c146c5ce7711, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733771128445 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f0f90ac5e7e4ae2af1f3aab89f7a6d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733771128485 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c8f9dc324d4ee2a17cf82381001b50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733771128485 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd2c6fd145546cdb79e72fdc93a2f54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733771129634 2024-12-09T19:05:30,718 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289e5befef414081bb73ed2cd65e0e77, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733771129633 2024-12-09T19:05:30,723 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:30,723 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:30,723 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/8084f8e3260f417786776255822f8990 is 50, key is test_row_0/B:col10/1733771129634/Put/seqid=0 2024-12-09T19:05:30,724 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b23954f90a1f48629e64152990e12626 is 50, key is test_row_0/A:col10/1733771129634/Put/seqid=0 2024-12-09T19:05:30,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742200_1376 (size=12104) 2024-12-09T19:05:30,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742199_1375 (size=12104) 2024-12-09T19:05:30,733 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b23954f90a1f48629e64152990e12626 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b23954f90a1f48629e64152990e12626 2024-12-09T19:05:30,737 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into b23954f90a1f48629e64152990e12626(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:30,737 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:30,737 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=13, startTime=1733771130716; duration=0sec 2024-12-09T19:05:30,737 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:30,737 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:30,737 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:30,738 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:30,738 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:30,738 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:30,738 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/94b32b1cf17a44fbbbcdb46c3a5d1237, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/8b0bad5f6c394ae7addf898bd2e82e2c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/be5eda9fde054c72b22f7bb6a892bd48] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=35.2 K 2024-12-09T19:05:30,738 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b32b1cf17a44fbbbcdb46c3a5d1237, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733771128445 2024-12-09T19:05:30,738 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b0bad5f6c394ae7addf898bd2e82e2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733771128485 2024-12-09T19:05:30,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T19:05:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:30,739 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:05:30,739 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting be5eda9fde054c72b22f7bb6a892bd48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733771129634 2024-12-09T19:05:30,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/1e274fdb4a6b4658bba651c766e650d4 is 50, key is test_row_0/A:col10/1733771130299/Put/seqid=0 2024-12-09T19:05:30,744 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:30,745 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/78a57c6b0e5048a0b19720b4314cb8bb is 50, key is test_row_0/C:col10/1733771129634/Put/seqid=0 2024-12-09T19:05:30,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742201_1377 (size=12001) 2024-12-09T19:05:30,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742202_1378 (size=12104) 2024-12-09T19:05:30,758 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/78a57c6b0e5048a0b19720b4314cb8bb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/78a57c6b0e5048a0b19720b4314cb8bb 2024-12-09T19:05:30,763 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into 78a57c6b0e5048a0b19720b4314cb8bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:30,763 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:30,763 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=13, startTime=1733771130717; duration=0sec 2024-12-09T19:05:30,763 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:30,763 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T19:05:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:30,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771190947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771190948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771190948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771190951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771190951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771191054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771191064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771191064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771191064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771191064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,134 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/8084f8e3260f417786776255822f8990 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/8084f8e3260f417786776255822f8990 2024-12-09T19:05:31,137 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 8084f8e3260f417786776255822f8990(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:31,137 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:31,137 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=13, startTime=1733771130717; duration=0sec 2024-12-09T19:05:31,137 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:31,137 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:31,146 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/1e274fdb4a6b4658bba651c766e650d4 2024-12-09T19:05:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/74d1dca6ba6c429fb7794ba75a1312b0 is 50, key is test_row_0/B:col10/1733771130299/Put/seqid=0 2024-12-09T19:05:31,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742203_1379 (size=12001) 2024-12-09T19:05:31,157 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/74d1dca6ba6c429fb7794ba75a1312b0 2024-12-09T19:05:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/2b1cb78029eb4c48a2bff2919f3fda5c is 50, key is test_row_0/C:col10/1733771130299/Put/seqid=0 2024-12-09T19:05:31,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742204_1380 (size=12001) 2024-12-09T19:05:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T19:05:31,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771191264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771191274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771191274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771191274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771191275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:05:31,568 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/2b1cb78029eb4c48a2bff2919f3fda5c 2024-12-09T19:05:31,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/1e274fdb4a6b4658bba651c766e650d4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/1e274fdb4a6b4658bba651c766e650d4 2024-12-09T19:05:31,574 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/1e274fdb4a6b4658bba651c766e650d4, entries=150, sequenceid=75, filesize=11.7 K 2024-12-09T19:05:31,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/74d1dca6ba6c429fb7794ba75a1312b0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/74d1dca6ba6c429fb7794ba75a1312b0 2024-12-09T19:05:31,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/74d1dca6ba6c429fb7794ba75a1312b0, entries=150, sequenceid=75, filesize=11.7 K 2024-12-09T19:05:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771191574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/2b1cb78029eb4c48a2bff2919f3fda5c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/2b1cb78029eb4c48a2bff2919f3fda5c 2024-12-09T19:05:31,583 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/2b1cb78029eb4c48a2bff2919f3fda5c, entries=150, sequenceid=75, filesize=11.7 K 2024-12-09T19:05:31,583 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e7537c2a1583e00ca6f87dcf9620724c in 844ms, sequenceid=75, compaction requested=false 2024-12-09T19:05:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-09T19:05:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-09T19:05:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:31,588 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:31,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-09T19:05:31,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-09T19:05:31,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 998 msec 2024-12-09T19:05:31,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0040 sec 2024-12-09T19:05:31,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/325c322230b141dbb5e472588f59e897 is 50, key is test_row_0/A:col10/1733771130946/Put/seqid=0 2024-12-09T19:05:31,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742205_1381 (size=14341) 2024-12-09T19:05:31,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/325c322230b141dbb5e472588f59e897 2024-12-09T19:05:31,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/0d2215c5862f4a23b56d0e0d56371268 is 50, key is test_row_0/B:col10/1733771130946/Put/seqid=0 2024-12-09T19:05:31,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742206_1382 (size=12001) 2024-12-09T19:05:31,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771191624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771191624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771191627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771191628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T19:05:31,724 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:31,724 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-09T19:05:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:31,726 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:31,727 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:31,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771191734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771191734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771191744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771191744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:31,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:31,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:31,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771191944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771191944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771191954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771191954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/0d2215c5862f4a23b56d0e0d56371268 2024-12-09T19:05:32,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/49cd15d573744f15b2b5e7f65e014c0b is 50, key is test_row_0/C:col10/1733771130946/Put/seqid=0 2024-12-09T19:05:32,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742207_1383 (size=12001) 2024-12-09T19:05:32,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/49cd15d573744f15b2b5e7f65e014c0b 2024-12-09T19:05:32,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/325c322230b141dbb5e472588f59e897 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/325c322230b141dbb5e472588f59e897 2024-12-09T19:05:32,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/325c322230b141dbb5e472588f59e897, entries=200, sequenceid=91, filesize=14.0 K 2024-12-09T19:05:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/0d2215c5862f4a23b56d0e0d56371268 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/0d2215c5862f4a23b56d0e0d56371268 2024-12-09T19:05:32,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/0d2215c5862f4a23b56d0e0d56371268, entries=150, sequenceid=91, filesize=11.7 K 2024-12-09T19:05:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/49cd15d573744f15b2b5e7f65e014c0b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/49cd15d573744f15b2b5e7f65e014c0b 2024-12-09T19:05:32,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/49cd15d573744f15b2b5e7f65e014c0b, entries=150, sequenceid=91, filesize=11.7 K 2024-12-09T19:05:32,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e7537c2a1583e00ca6f87dcf9620724c in 455ms, sequenceid=91, compaction requested=true 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:32,043 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T19:05:32,043 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:32,044 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:32,044 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:32,044 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,045 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b23954f90a1f48629e64152990e12626, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/1e274fdb4a6b4658bba651c766e650d4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/325c322230b141dbb5e472588f59e897] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=37.5 K 2024-12-09T19:05:32,045 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:32,045 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:32,045 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,045 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/8084f8e3260f417786776255822f8990, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/74d1dca6ba6c429fb7794ba75a1312b0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/0d2215c5862f4a23b56d0e0d56371268] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=35.3 K 2024-12-09T19:05:32,045 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b23954f90a1f48629e64152990e12626, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733771129634 2024-12-09T19:05:32,045 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e274fdb4a6b4658bba651c766e650d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733771130297 2024-12-09T19:05:32,045 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8084f8e3260f417786776255822f8990, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733771129634 2024-12-09T19:05:32,046 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325c322230b141dbb5e472588f59e897, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733771130946 2024-12-09T19:05:32,046 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d1dca6ba6c429fb7794ba75a1312b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733771130297 2024-12-09T19:05:32,046 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2215c5862f4a23b56d0e0d56371268, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733771130946 2024-12-09T19:05:32,052 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:32,052 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5c6fc32fc73b4a31ac3b360a0062b953 is 50, key is test_row_0/B:col10/1733771130946/Put/seqid=0 2024-12-09T19:05:32,052 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:32,053 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/3e0540427354464b9d00697001ded3b8 is 50, key is test_row_0/A:col10/1733771130946/Put/seqid=0 2024-12-09T19:05:32,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742208_1384 (size=12207) 2024-12-09T19:05:32,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742209_1385 (size=12207) 2024-12-09T19:05:32,069 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/3e0540427354464b9d00697001ded3b8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3e0540427354464b9d00697001ded3b8 2024-12-09T19:05:32,077 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into 3e0540427354464b9d00697001ded3b8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:32,077 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:32,078 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=13, startTime=1733771132043; duration=0sec 2024-12-09T19:05:32,078 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:32,078 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:32,078 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:32,079 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:32,079 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:32,079 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,080 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/78a57c6b0e5048a0b19720b4314cb8bb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/2b1cb78029eb4c48a2bff2919f3fda5c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/49cd15d573744f15b2b5e7f65e014c0b] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=35.3 K 2024-12-09T19:05:32,080 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78a57c6b0e5048a0b19720b4314cb8bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733771129634 2024-12-09T19:05:32,080 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1cb78029eb4c48a2bff2919f3fda5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733771130297 2024-12-09T19:05:32,080 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cd15d573744f15b2b5e7f65e014c0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733771130946 2024-12-09T19:05:32,086 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:32,086 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/15a3733f9aa442b19a328b54dee2765d is 50, key is test_row_0/C:col10/1733771130946/Put/seqid=0 2024-12-09T19:05:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742210_1386 (size=12207) 2024-12-09T19:05:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:32,099 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:32,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:05:32,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e7e614070cd54dd98dbffbdb48af2011 is 50, key is test_row_0/A:col10/1733771132098/Put/seqid=0 2024-12-09T19:05:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742211_1387 (size=14341) 2024-12-09T19:05:32,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e7e614070cd54dd98dbffbdb48af2011 2024-12-09T19:05:32,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/a8dc94214f3149d4902365fd1f909319 is 50, key is test_row_0/B:col10/1733771132098/Put/seqid=0 2024-12-09T19:05:32,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742212_1388 (size=12001) 2024-12-09T19:05:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771192162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771192255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771192255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771192264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771192264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771192274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:32,462 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5c6fc32fc73b4a31ac3b360a0062b953 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5c6fc32fc73b4a31ac3b360a0062b953 2024-12-09T19:05:32,466 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 5c6fc32fc73b4a31ac3b360a0062b953(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:32,466 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:32,466 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=13, startTime=1733771132043; duration=0sec 2024-12-09T19:05:32,466 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:32,466 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:32,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771192484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,497 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/15a3733f9aa442b19a328b54dee2765d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/15a3733f9aa442b19a328b54dee2765d 2024-12-09T19:05:32,501 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into 15a3733f9aa442b19a328b54dee2765d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:32,501 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:32,501 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=13, startTime=1733771132043; duration=0sec 2024-12-09T19:05:32,501 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:32,501 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:32,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/a8dc94214f3149d4902365fd1f909319 2024-12-09T19:05:32,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/44196552d0b44e29aee0d0baea1ca913 is 50, key is test_row_0/C:col10/1733771132098/Put/seqid=0 2024-12-09T19:05:32,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742213_1389 (size=12001) 2024-12-09T19:05:32,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771192764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771192774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771192774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771192774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771192795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:32,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/44196552d0b44e29aee0d0baea1ca913 2024-12-09T19:05:32,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e7e614070cd54dd98dbffbdb48af2011 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e7e614070cd54dd98dbffbdb48af2011 2024-12-09T19:05:32,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e7e614070cd54dd98dbffbdb48af2011, entries=200, sequenceid=115, filesize=14.0 K 2024-12-09T19:05:32,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/a8dc94214f3149d4902365fd1f909319 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8dc94214f3149d4902365fd1f909319 2024-12-09T19:05:32,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8dc94214f3149d4902365fd1f909319, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:05:32,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:32,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/44196552d0b44e29aee0d0baea1ca913 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/44196552d0b44e29aee0d0baea1ca913 2024-12-09T19:05:32,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:32,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:32,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:32,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/44196552d0b44e29aee0d0baea1ca913, entries=150, sequenceid=115, filesize=11.7 K 2024-12-09T19:05:32,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e7537c2a1583e00ca6f87dcf9620724c in 853ms, sequenceid=115, compaction requested=false 2024-12-09T19:05:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:33,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-09T19:05:33,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:33,100 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:05:33,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ba7512ca6cb8457aa7a8ef0c523aac90 is 50, key is test_row_0/A:col10/1733771132160/Put/seqid=0 2024-12-09T19:05:33,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742214_1390 (size=12001) 2024-12-09T19:05:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771193431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,509 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ba7512ca6cb8457aa7a8ef0c523aac90 2024-12-09T19:05:33,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/88a041a250154d58a806f43052b9d17a is 50, key is test_row_0/B:col10/1733771132160/Put/seqid=0 2024-12-09T19:05:33,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742215_1391 (size=12001) 2024-12-09T19:05:33,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771193544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771193754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771193784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771193785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771193785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,789 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in e7537c2a1583e00ca6f87dcf9620724c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,790 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in e7537c2a1583e00ca6f87dcf9620724c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,790 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in e7537c2a1583e00ca6f87dcf9620724c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771193794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:33,798 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in e7537c2a1583e00ca6f87dcf9620724c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:05:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:33,918 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/88a041a250154d58a806f43052b9d17a 2024-12-09T19:05:33,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/eedcdc298477425491cf74848aef56dc is 50, key is test_row_0/C:col10/1733771132160/Put/seqid=0 2024-12-09T19:05:33,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742216_1392 (size=12001) 2024-12-09T19:05:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771194074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:34,331 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/eedcdc298477425491cf74848aef56dc 2024-12-09T19:05:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ba7512ca6cb8457aa7a8ef0c523aac90 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ba7512ca6cb8457aa7a8ef0c523aac90 2024-12-09T19:05:34,338 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ba7512ca6cb8457aa7a8ef0c523aac90, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T19:05:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/88a041a250154d58a806f43052b9d17a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/88a041a250154d58a806f43052b9d17a 2024-12-09T19:05:34,342 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/88a041a250154d58a806f43052b9d17a, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T19:05:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/eedcdc298477425491cf74848aef56dc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eedcdc298477425491cf74848aef56dc 2024-12-09T19:05:34,346 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eedcdc298477425491cf74848aef56dc, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T19:05:34,346 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e7537c2a1583e00ca6f87dcf9620724c in 1246ms, sequenceid=130, compaction requested=true 2024-12-09T19:05:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:34,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-09T19:05:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-09T19:05:34,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T19:05:34,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6210 sec 2024-12-09T19:05:34,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.6240 sec 2024-12-09T19:05:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:34,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:05:34,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/7f2bd9c4798e44638eab4490d7727b70 is 50, key is test_row_0/A:col10/1733771133430/Put/seqid=0 2024-12-09T19:05:34,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742217_1393 (size=14541) 2024-12-09T19:05:34,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/7f2bd9c4798e44638eab4490d7727b70 2024-12-09T19:05:34,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/bd07acfebf6c454098bfa15f9dca1428 is 50, key is test_row_0/B:col10/1733771133430/Put/seqid=0 2024-12-09T19:05:34,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742218_1394 (size=12151) 2024-12-09T19:05:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771194657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771194775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:34,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771194984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/bd07acfebf6c454098bfa15f9dca1428 2024-12-09T19:05:35,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/1ee48f360a9f4b57b3479e83cbb5d4fc is 50, key is test_row_0/C:col10/1733771133430/Put/seqid=0 2024-12-09T19:05:35,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742219_1395 (size=12151) 2024-12-09T19:05:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771195295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/1ee48f360a9f4b57b3479e83cbb5d4fc 2024-12-09T19:05:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/7f2bd9c4798e44638eab4490d7727b70 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7f2bd9c4798e44638eab4490d7727b70 2024-12-09T19:05:35,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7f2bd9c4798e44638eab4490d7727b70, entries=200, sequenceid=154, filesize=14.2 K 2024-12-09T19:05:35,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/bd07acfebf6c454098bfa15f9dca1428 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/bd07acfebf6c454098bfa15f9dca1428 2024-12-09T19:05:35,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/bd07acfebf6c454098bfa15f9dca1428, entries=150, sequenceid=154, filesize=11.9 K 2024-12-09T19:05:35,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/1ee48f360a9f4b57b3479e83cbb5d4fc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/1ee48f360a9f4b57b3479e83cbb5d4fc 2024-12-09T19:05:35,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/1ee48f360a9f4b57b3479e83cbb5d4fc, entries=150, sequenceid=154, filesize=11.9 K 2024-12-09T19:05:35,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e7537c2a1583e00ca6f87dcf9620724c in 842ms, sequenceid=154, compaction requested=true 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:35,430 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:35,430 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:35,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:35,431 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:35,431 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:35,431 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:35,431 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:35,431 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:35,431 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:35,431 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3e0540427354464b9d00697001ded3b8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e7e614070cd54dd98dbffbdb48af2011, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ba7512ca6cb8457aa7a8ef0c523aac90, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7f2bd9c4798e44638eab4490d7727b70] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=51.8 K 2024-12-09T19:05:35,431 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5c6fc32fc73b4a31ac3b360a0062b953, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8dc94214f3149d4902365fd1f909319, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/88a041a250154d58a806f43052b9d17a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/bd07acfebf6c454098bfa15f9dca1428] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=47.2 K 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0540427354464b9d00697001ded3b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733771130946 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c6fc32fc73b4a31ac3b360a0062b953, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733771130946 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a8dc94214f3149d4902365fd1f909319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771131622 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e614070cd54dd98dbffbdb48af2011, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771131622 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba7512ca6cb8457aa7a8ef0c523aac90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771132136 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a041a250154d58a806f43052b9d17a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771132136 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting bd07acfebf6c454098bfa15f9dca1428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771133398 2024-12-09T19:05:35,432 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2bd9c4798e44638eab4490d7727b70, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771133398 2024-12-09T19:05:35,438 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:35,438 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:35,439 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b97cd4ed71a24108955791d5ac20121e is 50, key is test_row_0/A:col10/1733771133430/Put/seqid=0 2024-12-09T19:05:35,439 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/b64b47a941e04c2d9866bbaeff9aa18b is 50, key is test_row_0/B:col10/1733771133430/Put/seqid=0 2024-12-09T19:05:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742221_1397 (size=12493) 2024-12-09T19:05:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742220_1396 (size=12493) 2024-12-09T19:05:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:35,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:35,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:05:35,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b9787c9db2ef408cb1065cb797793c63 is 50, key is test_row_0/A:col10/1733771134628/Put/seqid=0 2024-12-09T19:05:35,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742222_1398 (size=12151) 2024-12-09T19:05:35,851 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/b64b47a941e04c2d9866bbaeff9aa18b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b64b47a941e04c2d9866bbaeff9aa18b 2024-12-09T19:05:35,852 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b97cd4ed71a24108955791d5ac20121e as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b97cd4ed71a24108955791d5ac20121e 2024-12-09T19:05:35,855 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into b64b47a941e04c2d9866bbaeff9aa18b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:35,855 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=12, startTime=1733771135430; duration=0sec 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:35,855 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into b97cd4ed71a24108955791d5ac20121e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:35,855 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=12, startTime=1733771135430; duration=0sec 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:35,855 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:35,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:35,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:35,856 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:35,856 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/15a3733f9aa442b19a328b54dee2765d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/44196552d0b44e29aee0d0baea1ca913, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eedcdc298477425491cf74848aef56dc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/1ee48f360a9f4b57b3479e83cbb5d4fc] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=47.2 K 2024-12-09T19:05:35,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a3733f9aa442b19a328b54dee2765d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733771130946 2024-12-09T19:05:35,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 44196552d0b44e29aee0d0baea1ca913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733771131622 2024-12-09T19:05:35,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771195846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771195847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,857 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting eedcdc298477425491cf74848aef56dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733771132136 2024-12-09T19:05:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,858 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ee48f360a9f4b57b3479e83cbb5d4fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771133398 2024-12-09T19:05:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771195848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771195849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,864 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:35,865 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/9059a872456246808fe45b5202e47f36 is 50, key is test_row_0/C:col10/1733771133430/Put/seqid=0 2024-12-09T19:05:35,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771195857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742223_1399 (size=12493) 2024-12-09T19:05:35,878 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/9059a872456246808fe45b5202e47f36 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9059a872456246808fe45b5202e47f36 2024-12-09T19:05:35,882 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into 9059a872456246808fe45b5202e47f36(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:35,882 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:35,882 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=12, startTime=1733771135430; duration=0sec 2024-12-09T19:05:35,882 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:35,882 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T19:05:35,895 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:35,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-09T19:05:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T19:05:35,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:35,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:35,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771195964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771195964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771195964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771195964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:35,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771195975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:35,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:35,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T19:05:36,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T19:05:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771196174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771196174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771196175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771196175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771196184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T19:05:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T19:05:36,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b9787c9db2ef408cb1065cb797793c63 2024-12-09T19:05:36,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/e6af3e229d9f4701ae219a3b9908a715 is 50, key is test_row_0/B:col10/1733771134628/Put/seqid=0 2024-12-09T19:05:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742224_1400 (size=12151) 2024-12-09T19:05:36,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T19:05:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771196484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771196484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771196484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771196484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771196495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:36,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:36,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T19:05:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T19:05:36,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/e6af3e229d9f4701ae219a3b9908a715 2024-12-09T19:05:36,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/87f6b416ddc7459abd3fb47f1449c84c is 50, key is test_row_0/C:col10/1733771134628/Put/seqid=0 2024-12-09T19:05:36,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742225_1401 (size=12151) 2024-12-09T19:05:36,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/87f6b416ddc7459abd3fb47f1449c84c 2024-12-09T19:05:36,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b9787c9db2ef408cb1065cb797793c63 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b9787c9db2ef408cb1065cb797793c63 2024-12-09T19:05:36,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b9787c9db2ef408cb1065cb797793c63, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T19:05:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/e6af3e229d9f4701ae219a3b9908a715 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e6af3e229d9f4701ae219a3b9908a715 2024-12-09T19:05:36,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e6af3e229d9f4701ae219a3b9908a715, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T19:05:36,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/87f6b416ddc7459abd3fb47f1449c84c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/87f6b416ddc7459abd3fb47f1449c84c 2024-12-09T19:05:36,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/87f6b416ddc7459abd3fb47f1449c84c, entries=150, sequenceid=168, filesize=11.9 K 2024-12-09T19:05:36,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e7537c2a1583e00ca6f87dcf9620724c in 845ms, sequenceid=168, compaction requested=false 2024-12-09T19:05:36,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:36,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-09T19:05:36,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:36,659 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-09T19:05:36,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/da38bf57641640f9bf5ed3a0663cf066 is 50, key is test_row_0/A:col10/1733771135856/Put/seqid=0 2024-12-09T19:05:36,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742226_1402 (size=12151) 2024-12-09T19:05:36,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:37,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771197008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771197010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771197016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771197017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771197017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T19:05:37,073 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/da38bf57641640f9bf5ed3a0663cf066 2024-12-09T19:05:37,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/c7e78f0256774d21ba7d053a4f0280af is 50, key is test_row_0/B:col10/1733771135856/Put/seqid=0 2024-12-09T19:05:37,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742227_1403 (size=12151) 2024-12-09T19:05:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771197124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771197124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771197124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771197135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771197135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771197334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771197334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771197334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771197345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771197345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/c7e78f0256774d21ba7d053a4f0280af 2024-12-09T19:05:37,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/b03fa62bc470466191fba47f24b39fd0 is 50, key is test_row_0/C:col10/1733771135856/Put/seqid=0 2024-12-09T19:05:37,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742228_1404 (size=12151) 2024-12-09T19:05:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771197644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771197644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771197644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771197654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771197655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:37,893 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/b03fa62bc470466191fba47f24b39fd0 2024-12-09T19:05:37,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/da38bf57641640f9bf5ed3a0663cf066 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/da38bf57641640f9bf5ed3a0663cf066 2024-12-09T19:05:37,899 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/da38bf57641640f9bf5ed3a0663cf066, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T19:05:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/c7e78f0256774d21ba7d053a4f0280af as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/c7e78f0256774d21ba7d053a4f0280af 2024-12-09T19:05:37,903 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/c7e78f0256774d21ba7d053a4f0280af, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T19:05:37,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/b03fa62bc470466191fba47f24b39fd0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/b03fa62bc470466191fba47f24b39fd0 2024-12-09T19:05:37,906 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/b03fa62bc470466191fba47f24b39fd0, entries=150, sequenceid=193, filesize=11.9 K 2024-12-09T19:05:37,907 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for e7537c2a1583e00ca6f87dcf9620724c in 1248ms, sequenceid=193, compaction requested=true 2024-12-09T19:05:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-09T19:05:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-09T19:05:37,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T19:05:37,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0120 sec 2024-12-09T19:05:37,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0150 sec 2024-12-09T19:05:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-09T19:05:38,055 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:38,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-09T19:05:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:38,057 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:38,057 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:38,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:38,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-09T19:05:38,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ac774bd5e73340d79fe4a2402deab2bc is 50, key is test_row_0/A:col10/1733771138157/Put/seqid=0 2024-12-09T19:05:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:38,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742229_1405 (size=16931) 2024-12-09T19:05:38,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771198180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771198181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771198182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771198184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771198184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771198294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771198294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771198294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771198294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771198295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771198504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771198504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771198505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771198505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771198505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ac774bd5e73340d79fe4a2402deab2bc 2024-12-09T19:05:38,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1e58014881fe449ca0fef1ae0a11a642 is 50, key is test_row_0/B:col10/1733771138157/Put/seqid=0 2024-12-09T19:05:38,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742230_1406 (size=12151) 2024-12-09T19:05:38,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771198814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771198814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771198815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771198815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771198815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:38,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:38,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1e58014881fe449ca0fef1ae0a11a642 2024-12-09T19:05:38,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/eae17031f8634342b80dd8ae797774a4 is 50, key is test_row_0/C:col10/1733771138157/Put/seqid=0 2024-12-09T19:05:38,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742231_1407 (size=12151) 2024-12-09T19:05:39,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:39,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:39,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771199324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771199324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:39,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771199325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:39,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771199325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:39,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771199334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:39,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/eae17031f8634342b80dd8ae797774a4 2024-12-09T19:05:39,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ac774bd5e73340d79fe4a2402deab2bc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ac774bd5e73340d79fe4a2402deab2bc 2024-12-09T19:05:39,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ac774bd5e73340d79fe4a2402deab2bc, entries=250, sequenceid=210, filesize=16.5 K 2024-12-09T19:05:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1e58014881fe449ca0fef1ae0a11a642 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1e58014881fe449ca0fef1ae0a11a642 2024-12-09T19:05:39,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1e58014881fe449ca0fef1ae0a11a642, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T19:05:39,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/eae17031f8634342b80dd8ae797774a4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eae17031f8634342b80dd8ae797774a4 2024-12-09T19:05:39,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eae17031f8634342b80dd8ae797774a4, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T19:05:39,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e7537c2a1583e00ca6f87dcf9620724c in 1245ms, sequenceid=210, compaction requested=true 2024-12-09T19:05:39,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:39,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:39,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:39,404 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:39,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:39,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:39,404 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:39,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:39,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:39,405 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:39,405 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:39,405 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,405 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:39,405 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:39,405 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b64b47a941e04c2d9866bbaeff9aa18b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e6af3e229d9f4701ae219a3b9908a715, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/c7e78f0256774d21ba7d053a4f0280af, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1e58014881fe449ca0fef1ae0a11a642] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=47.8 K 2024-12-09T19:05:39,405 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,405 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b97cd4ed71a24108955791d5ac20121e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b9787c9db2ef408cb1065cb797793c63, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/da38bf57641640f9bf5ed3a0663cf066, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ac774bd5e73340d79fe4a2402deab2bc] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=52.5 K 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97cd4ed71a24108955791d5ac20121e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771133398 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b64b47a941e04c2d9866bbaeff9aa18b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771133398 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e6af3e229d9f4701ae219a3b9908a715, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733771134628 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9787c9db2ef408cb1065cb797793c63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733771134628 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e78f0256774d21ba7d053a4f0280af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733771135845 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting da38bf57641640f9bf5ed3a0663cf066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733771135845 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e58014881fe449ca0fef1ae0a11a642, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771137007 2024-12-09T19:05:39,406 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac774bd5e73340d79fe4a2402deab2bc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771137007 2024-12-09T19:05:39,419 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:39,420 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5b07600e1e794f6e85c99963c799befe is 50, key is test_row_0/B:col10/1733771138157/Put/seqid=0 2024-12-09T19:05:39,420 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:39,421 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ef1e78e5cbc348b99137ee5584d9d7e0 is 50, key is test_row_0/A:col10/1733771138157/Put/seqid=0 2024-12-09T19:05:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742232_1408 (size=12629) 2024-12-09T19:05:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742233_1409 (size=12629) 2024-12-09T19:05:39,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-09T19:05:39,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,430 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-09T19:05:39,434 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/5b07600e1e794f6e85c99963c799befe as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5b07600e1e794f6e85c99963c799befe 2024-12-09T19:05:39,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e19cc4b9354a4a8bba460cbc27e82c8f is 50, key is test_row_0/A:col10/1733771138181/Put/seqid=0 2024-12-09T19:05:39,440 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 5b07600e1e794f6e85c99963c799befe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:39,440 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:39,440 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=12, startTime=1733771139404; duration=0sec 2024-12-09T19:05:39,441 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:39,441 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:39,441 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:05:39,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742234_1410 (size=12151) 2024-12-09T19:05:39,441 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e19cc4b9354a4a8bba460cbc27e82c8f 2024-12-09T19:05:39,443 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:05:39,443 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:39,443 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:39,443 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9059a872456246808fe45b5202e47f36, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/87f6b416ddc7459abd3fb47f1449c84c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/b03fa62bc470466191fba47f24b39fd0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eae17031f8634342b80dd8ae797774a4] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=47.8 K 2024-12-09T19:05:39,444 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9059a872456246808fe45b5202e47f36, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733771133398 2024-12-09T19:05:39,444 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f6b416ddc7459abd3fb47f1449c84c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733771134628 2024-12-09T19:05:39,445 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b03fa62bc470466191fba47f24b39fd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733771135845 2024-12-09T19:05:39,445 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting eae17031f8634342b80dd8ae797774a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771137007 2024-12-09T19:05:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/62968ad22622447f915773e63e4ede27 is 50, key is test_row_0/B:col10/1733771138181/Put/seqid=0 2024-12-09T19:05:39,462 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:39,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742235_1411 (size=12151) 2024-12-09T19:05:39,462 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/c3687973fc32437ea5c23e0ec4627358 is 50, key is test_row_0/C:col10/1733771138157/Put/seqid=0 2024-12-09T19:05:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742236_1412 (size=12629) 2024-12-09T19:05:39,832 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/ef1e78e5cbc348b99137ee5584d9d7e0 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ef1e78e5cbc348b99137ee5584d9d7e0 2024-12-09T19:05:39,836 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into ef1e78e5cbc348b99137ee5584d9d7e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:39,836 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:39,836 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=12, startTime=1733771139404; duration=0sec 2024-12-09T19:05:39,836 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:39,836 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:39,861 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/62968ad22622447f915773e63e4ede27 2024-12-09T19:05:39,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/544e9ec437ea46f0892edb4e4bcce0f1 is 50, key is test_row_0/C:col10/1733771138181/Put/seqid=0 2024-12-09T19:05:39,871 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/c3687973fc32437ea5c23e0ec4627358 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c3687973fc32437ea5c23e0ec4627358 2024-12-09T19:05:39,875 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into c3687973fc32437ea5c23e0ec4627358(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:39,875 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:39,875 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=12, startTime=1733771139404; duration=0sec 2024-12-09T19:05:39,875 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:39,875 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:39,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742237_1413 (size=12151) 2024-12-09T19:05:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:40,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/544e9ec437ea46f0892edb4e4bcce0f1 2024-12-09T19:05:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e19cc4b9354a4a8bba460cbc27e82c8f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e19cc4b9354a4a8bba460cbc27e82c8f 2024-12-09T19:05:40,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e19cc4b9354a4a8bba460cbc27e82c8f, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T19:05:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/62968ad22622447f915773e63e4ede27 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/62968ad22622447f915773e63e4ede27 2024-12-09T19:05:40,292 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/62968ad22622447f915773e63e4ede27, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T19:05:40,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/544e9ec437ea46f0892edb4e4bcce0f1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/544e9ec437ea46f0892edb4e4bcce0f1 2024-12-09T19:05:40,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/544e9ec437ea46f0892edb4e4bcce0f1, entries=150, sequenceid=229, filesize=11.9 K 2024-12-09T19:05:40,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for e7537c2a1583e00ca6f87dcf9620724c in 871ms, sequenceid=229, compaction requested=false 2024-12-09T19:05:40,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:40,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:40,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-09T19:05:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-09T19:05:40,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-09T19:05:40,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2450 sec 2024-12-09T19:05:40,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.2490 sec 2024-12-09T19:05:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:40,343 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:40,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:05:40,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/40544b5c201f4d3c9e87fe5ab0711d0d is 50, key is test_row_0/A:col10/1733771140338/Put/seqid=0 2024-12-09T19:05:40,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742238_1414 (size=14541) 2024-12-09T19:05:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771200373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771200374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771200374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771200375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771200381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771200484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771200484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771200494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771200494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771200494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771200694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771200694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771200704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771200704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771200704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:40,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/40544b5c201f4d3c9e87fe5ab0711d0d 2024-12-09T19:05:40,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/04632d092d7d462b8b4ee53d4f9c2f2f is 50, key is test_row_0/B:col10/1733771140338/Put/seqid=0 2024-12-09T19:05:40,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742239_1415 (size=12151) 2024-12-09T19:05:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771201005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771201005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771201014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771201014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771201014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/04632d092d7d462b8b4ee53d4f9c2f2f 2024-12-09T19:05:41,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/f356109a60a64cbebd9ec74c247339a4 is 50, key is test_row_0/C:col10/1733771140338/Put/seqid=0 2024-12-09T19:05:41,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742240_1416 (size=12151) 2024-12-09T19:05:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771201524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771201524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771201524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771201524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771201524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:41,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/f356109a60a64cbebd9ec74c247339a4 2024-12-09T19:05:41,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/40544b5c201f4d3c9e87fe5ab0711d0d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/40544b5c201f4d3c9e87fe5ab0711d0d 2024-12-09T19:05:41,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/40544b5c201f4d3c9e87fe5ab0711d0d, entries=200, sequenceid=243, filesize=14.2 K 2024-12-09T19:05:41,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/04632d092d7d462b8b4ee53d4f9c2f2f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/04632d092d7d462b8b4ee53d4f9c2f2f 2024-12-09T19:05:41,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/04632d092d7d462b8b4ee53d4f9c2f2f, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T19:05:41,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/f356109a60a64cbebd9ec74c247339a4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f356109a60a64cbebd9ec74c247339a4 2024-12-09T19:05:41,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f356109a60a64cbebd9ec74c247339a4, entries=150, sequenceid=243, filesize=11.9 K 2024-12-09T19:05:41,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e7537c2a1583e00ca6f87dcf9620724c in 1240ms, sequenceid=243, compaction requested=true 2024-12-09T19:05:41,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:41,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:41,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:41,584 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:41,584 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:41,584 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:41,584 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:41,584 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:41,584 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:41,584 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:41,584 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:41,584 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ef1e78e5cbc348b99137ee5584d9d7e0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e19cc4b9354a4a8bba460cbc27e82c8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/40544b5c201f4d3c9e87fe5ab0711d0d] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=38.4 K 2024-12-09T19:05:41,584 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5b07600e1e794f6e85c99963c799befe, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/62968ad22622447f915773e63e4ede27, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/04632d092d7d462b8b4ee53d4f9c2f2f] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.1 K 2024-12-09T19:05:41,585 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b07600e1e794f6e85c99963c799befe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771137007 2024-12-09T19:05:41,585 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1e78e5cbc348b99137ee5584d9d7e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771137007 2024-12-09T19:05:41,585 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e19cc4b9354a4a8bba460cbc27e82c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733771138180 2024-12-09T19:05:41,585 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 62968ad22622447f915773e63e4ede27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733771138180 2024-12-09T19:05:41,585 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40544b5c201f4d3c9e87fe5ab0711d0d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733771140334 2024-12-09T19:05:41,585 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 04632d092d7d462b8b4ee53d4f9c2f2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733771140338 2024-12-09T19:05:41,590 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#354 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:41,590 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/34707b17a81242148dbb82eaed079d56 is 50, key is test_row_0/B:col10/1733771140338/Put/seqid=0 2024-12-09T19:05:41,591 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:41,592 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e60dfc3fd2ad4b2fbbc95e27177e60f5 is 50, key is test_row_0/A:col10/1733771140338/Put/seqid=0 2024-12-09T19:05:41,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742241_1417 (size=12731) 2024-12-09T19:05:41,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742242_1418 (size=12731) 2024-12-09T19:05:41,598 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/34707b17a81242148dbb82eaed079d56 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/34707b17a81242148dbb82eaed079d56 2024-12-09T19:05:41,603 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 34707b17a81242148dbb82eaed079d56(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:41,603 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:41,603 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=13, startTime=1733771141584; duration=0sec 2024-12-09T19:05:41,603 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:41,603 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:41,603 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:41,604 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:41,604 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:41,604 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:41,604 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c3687973fc32437ea5c23e0ec4627358, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/544e9ec437ea46f0892edb4e4bcce0f1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f356109a60a64cbebd9ec74c247339a4] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.1 K 2024-12-09T19:05:41,604 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting c3687973fc32437ea5c23e0ec4627358, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771137007 2024-12-09T19:05:41,604 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 544e9ec437ea46f0892edb4e4bcce0f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733771138180 2024-12-09T19:05:41,605 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f356109a60a64cbebd9ec74c247339a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733771140338 2024-12-09T19:05:41,610 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:41,610 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/11f4819cdc9041f9b0c4c6d9ac9d4efc is 50, key is test_row_0/C:col10/1733771140338/Put/seqid=0 2024-12-09T19:05:41,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742243_1419 (size=12731) 2024-12-09T19:05:42,051 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e60dfc3fd2ad4b2fbbc95e27177e60f5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e60dfc3fd2ad4b2fbbc95e27177e60f5 2024-12-09T19:05:42,051 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/11f4819cdc9041f9b0c4c6d9ac9d4efc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/11f4819cdc9041f9b0c4c6d9ac9d4efc 2024-12-09T19:05:42,056 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into e60dfc3fd2ad4b2fbbc95e27177e60f5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:42,056 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into 11f4819cdc9041f9b0c4c6d9ac9d4efc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:42,056 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:42,056 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:42,056 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=13, startTime=1733771141584; duration=0sec 2024-12-09T19:05:42,056 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=13, startTime=1733771141583; duration=0sec 2024-12-09T19:05:42,056 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:42,056 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:42,056 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:42,056 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T19:05:42,225 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:42,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-09T19:05:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T19:05:42,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:42,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:42,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T19:05:42,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-09T19:05:42,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:42,380 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:05:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/7bcb174f908743f180b63f8a37940cc5 is 50, key is test_row_0/A:col10/1733771140362/Put/seqid=0 2024-12-09T19:05:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742244_1420 (size=12301) 2024-12-09T19:05:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:42,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T19:05:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771202550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771202551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771202551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771202552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771202553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771202665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771202665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771202665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771202665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771202665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,788 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/7bcb174f908743f180b63f8a37940cc5 2024-12-09T19:05:42,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/993d24b239b2449e8152d09e32fbfc4b is 50, key is test_row_0/B:col10/1733771140362/Put/seqid=0 2024-12-09T19:05:42,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742245_1421 (size=12301) 2024-12-09T19:05:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T19:05:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771202874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771203185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771203185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771203185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771203185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771203185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,196 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/993d24b239b2449e8152d09e32fbfc4b 2024-12-09T19:05:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/55fa81ab44d040fba7d76e5084a2d8fa is 50, key is test_row_0/C:col10/1733771140362/Put/seqid=0 2024-12-09T19:05:43,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742246_1422 (size=12301) 2024-12-09T19:05:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T19:05:43,606 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/55fa81ab44d040fba7d76e5084a2d8fa 2024-12-09T19:05:43,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/7bcb174f908743f180b63f8a37940cc5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7bcb174f908743f180b63f8a37940cc5 2024-12-09T19:05:43,613 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7bcb174f908743f180b63f8a37940cc5, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T19:05:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/993d24b239b2449e8152d09e32fbfc4b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/993d24b239b2449e8152d09e32fbfc4b 2024-12-09T19:05:43,616 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/993d24b239b2449e8152d09e32fbfc4b, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T19:05:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/55fa81ab44d040fba7d76e5084a2d8fa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/55fa81ab44d040fba7d76e5084a2d8fa 2024-12-09T19:05:43,620 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/55fa81ab44d040fba7d76e5084a2d8fa, entries=150, sequenceid=271, filesize=12.0 K 2024-12-09T19:05:43,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e7537c2a1583e00ca6f87dcf9620724c in 1241ms, sequenceid=271, compaction requested=false 2024-12-09T19:05:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-09T19:05:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-09T19:05:43,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T19:05:43,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3940 sec 2024-12-09T19:05:43,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.3990 sec 2024-12-09T19:05:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:43,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:43,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:05:43,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/88a49f47bba24d959af74832ab4545b1 is 50, key is test_row_0/A:col10/1733771143695/Put/seqid=0 2024-12-09T19:05:43,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742247_1423 (size=14741) 2024-12-09T19:05:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771203719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771203725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771203726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771203726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771203728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771203834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771203834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771203834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771203835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771203844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771204044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771204045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771204045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771204045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771204054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/88a49f47bba24d959af74832ab4545b1 2024-12-09T19:05:44,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/e184a230255a4d00ba0b188766b2f768 is 50, key is test_row_0/B:col10/1733771143695/Put/seqid=0 2024-12-09T19:05:44,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742248_1424 (size=12301) 2024-12-09T19:05:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771204354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771204354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771204354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771204354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771204365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-09T19:05:44,374 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:44,374 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T19:05:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T19:05:44,376 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:44,377 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:44,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T19:05:44,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/e184a230255a4d00ba0b188766b2f768 2024-12-09T19:05:44,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T19:05:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/c2fd2e5edfd9469daa81750c1a5dc1b4 is 50, key is test_row_0/C:col10/1733771143695/Put/seqid=0 2024-12-09T19:05:44,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742249_1425 (size=12301) 2024-12-09T19:05:44,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T19:05:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T19:05:44,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T19:05:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771204864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771204865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771204874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771204874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771204874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:44,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/c2fd2e5edfd9469daa81750c1a5dc1b4 2024-12-09T19:05:44,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/88a49f47bba24d959af74832ab4545b1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/88a49f47bba24d959af74832ab4545b1 2024-12-09T19:05:44,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/88a49f47bba24d959af74832ab4545b1, entries=200, sequenceid=283, filesize=14.4 K 2024-12-09T19:05:44,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/e184a230255a4d00ba0b188766b2f768 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e184a230255a4d00ba0b188766b2f768 2024-12-09T19:05:44,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e184a230255a4d00ba0b188766b2f768, entries=150, sequenceid=283, filesize=12.0 K 2024-12-09T19:05:44,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/c2fd2e5edfd9469daa81750c1a5dc1b4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c2fd2e5edfd9469daa81750c1a5dc1b4 2024-12-09T19:05:44,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c2fd2e5edfd9469daa81750c1a5dc1b4, entries=150, sequenceid=283, filesize=12.0 K 2024-12-09T19:05:44,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e7537c2a1583e00ca6f87dcf9620724c in 1253ms, sequenceid=283, compaction requested=true 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:44,950 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:44,950 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:44,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:44,950 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:44,951 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:44,951 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e60dfc3fd2ad4b2fbbc95e27177e60f5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7bcb174f908743f180b63f8a37940cc5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/88a49f47bba24d959af74832ab4545b1] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=38.8 K 2024-12-09T19:05:44,951 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,951 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/34707b17a81242148dbb82eaed079d56, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/993d24b239b2449e8152d09e32fbfc4b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e184a230255a4d00ba0b188766b2f768] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.5 K 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60dfc3fd2ad4b2fbbc95e27177e60f5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733771140338 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 34707b17a81242148dbb82eaed079d56, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733771140338 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bcb174f908743f180b63f8a37940cc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733771140362 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 993d24b239b2449e8152d09e32fbfc4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733771140362 2024-12-09T19:05:44,951 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a49f47bba24d959af74832ab4545b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733771142550 2024-12-09T19:05:44,952 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting e184a230255a4d00ba0b188766b2f768, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733771142551 2024-12-09T19:05:44,956 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#363 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:44,957 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/77df9e474a5c459cac9999f7fb10a3e3 is 50, key is test_row_0/B:col10/1733771143695/Put/seqid=0 2024-12-09T19:05:44,959 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:44,959 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/51aa484316a64732a44e20d1d5092cad is 50, key is test_row_0/A:col10/1733771143695/Put/seqid=0 2024-12-09T19:05:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742250_1426 (size=12983) 2024-12-09T19:05:44,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742251_1427 (size=12983) 2024-12-09T19:05:44,967 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/77df9e474a5c459cac9999f7fb10a3e3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/77df9e474a5c459cac9999f7fb10a3e3 2024-12-09T19:05:44,970 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 77df9e474a5c459cac9999f7fb10a3e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:44,970 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:44,970 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=13, startTime=1733771144950; duration=0sec 2024-12-09T19:05:44,970 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:44,970 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:44,970 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:44,972 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:44,972 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:44,972 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,972 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/11f4819cdc9041f9b0c4c6d9ac9d4efc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/55fa81ab44d040fba7d76e5084a2d8fa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c2fd2e5edfd9469daa81750c1a5dc1b4] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.5 K 2024-12-09T19:05:44,973 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f4819cdc9041f9b0c4c6d9ac9d4efc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733771140338 2024-12-09T19:05:44,973 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 55fa81ab44d040fba7d76e5084a2d8fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733771140362 2024-12-09T19:05:44,973 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting c2fd2e5edfd9469daa81750c1a5dc1b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733771142551 2024-12-09T19:05:44,980 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#365 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:44,980 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/ec5d5bb657be4007bc8815f5c0c32c45 is 50, key is test_row_0/C:col10/1733771143695/Put/seqid=0 2024-12-09T19:05:44,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742252_1428 (size=12983) 2024-12-09T19:05:44,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T19:05:44,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:44,986 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:05:44,991 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/ec5d5bb657be4007bc8815f5c0c32c45 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ec5d5bb657be4007bc8815f5c0c32c45 2024-12-09T19:05:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/801c57d2f5474625933f46b5ce796c48 is 50, key is test_row_0/A:col10/1733771143718/Put/seqid=0 2024-12-09T19:05:44,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742253_1429 (size=12301) 2024-12-09T19:05:44,997 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into ec5d5bb657be4007bc8815f5c0c32c45(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:44,997 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:44,997 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=13, startTime=1733771144950; duration=0sec 2024-12-09T19:05:44,997 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:44,997 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T19:05:45,367 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/51aa484316a64732a44e20d1d5092cad as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/51aa484316a64732a44e20d1d5092cad 2024-12-09T19:05:45,372 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into 51aa484316a64732a44e20d1d5092cad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:45,373 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:45,373 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=13, startTime=1733771144950; duration=0sec 2024-12-09T19:05:45,373 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:45,373 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:45,398 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/801c57d2f5474625933f46b5ce796c48 2024-12-09T19:05:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/512a7b518d1141a79e45908a679afcc1 is 50, key is test_row_0/B:col10/1733771143718/Put/seqid=0 2024-12-09T19:05:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742254_1430 (size=12301) 2024-12-09T19:05:45,407 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/512a7b518d1141a79e45908a679afcc1 2024-12-09T19:05:45,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5 is 50, key is test_row_0/C:col10/1733771143718/Put/seqid=0 2024-12-09T19:05:45,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742255_1431 (size=12301) 2024-12-09T19:05:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T19:05:45,815 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5 2024-12-09T19:05:45,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/801c57d2f5474625933f46b5ce796c48 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/801c57d2f5474625933f46b5ce796c48 2024-12-09T19:05:45,828 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/801c57d2f5474625933f46b5ce796c48, entries=150, sequenceid=308, filesize=12.0 K 2024-12-09T19:05:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/512a7b518d1141a79e45908a679afcc1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/512a7b518d1141a79e45908a679afcc1 2024-12-09T19:05:45,832 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/512a7b518d1141a79e45908a679afcc1, entries=150, sequenceid=308, filesize=12.0 K 2024-12-09T19:05:45,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5 2024-12-09T19:05:45,836 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5, entries=150, sequenceid=308, filesize=12.0 K 2024-12-09T19:05:45,837 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for e7537c2a1583e00ca6f87dcf9620724c in 851ms, sequenceid=308, compaction requested=false 2024-12-09T19:05:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:45,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T19:05:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T19:05:45,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T19:05:45,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-12-09T19:05:45,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4670 sec 2024-12-09T19:05:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:45,894 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:45,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-09T19:05:45,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e8ff1734a66a473aba73f4e305bc4f34 is 50, key is test_row_0/A:col10/1733771145893/Put/seqid=0 2024-12-09T19:05:45,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742256_1432 (size=17181) 2024-12-09T19:05:45,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771205929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771205929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771205931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771205930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771205931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771206044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771206044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771206045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771206045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771206045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771206254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771206264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e8ff1734a66a473aba73f4e305bc4f34 2024-12-09T19:05:46,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/ce3e781a73ad4291928a280cdd0ca87c is 50, key is test_row_0/B:col10/1733771145893/Put/seqid=0 2024-12-09T19:05:46,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742257_1433 (size=12301) 2024-12-09T19:05:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T19:05:46,524 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:46,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T19:05:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T19:05:46,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:46,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:46,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:46,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771206564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771206564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771206564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771206575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771206575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T19:05:46,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T19:05:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:46,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,700 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T19:05:46,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/ce3e781a73ad4291928a280cdd0ca87c 2024-12-09T19:05:46,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/208e7a71e2f346a1aa5d5a3045e74000 is 50, key is test_row_0/C:col10/1733771145893/Put/seqid=0 2024-12-09T19:05:46,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742258_1434 (size=12301) 2024-12-09T19:05:46,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T19:05:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:46,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T19:05:46,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T19:05:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:46,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771207075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771207075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771207075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771207094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:47,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771207094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:47,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/208e7a71e2f346a1aa5d5a3045e74000 2024-12-09T19:05:47,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T19:05:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:47,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T19:05:47,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/e8ff1734a66a473aba73f4e305bc4f34 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e8ff1734a66a473aba73f4e305bc4f34 2024-12-09T19:05:47,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e8ff1734a66a473aba73f4e305bc4f34, entries=250, sequenceid=323, filesize=16.8 K 2024-12-09T19:05:47,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/ce3e781a73ad4291928a280cdd0ca87c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/ce3e781a73ad4291928a280cdd0ca87c 2024-12-09T19:05:47,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/ce3e781a73ad4291928a280cdd0ca87c, entries=150, sequenceid=323, filesize=12.0 K 2024-12-09T19:05:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/208e7a71e2f346a1aa5d5a3045e74000 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/208e7a71e2f346a1aa5d5a3045e74000 2024-12-09T19:05:47,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/208e7a71e2f346a1aa5d5a3045e74000, entries=150, sequenceid=323, filesize=12.0 K 2024-12-09T19:05:47,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for e7537c2a1583e00ca6f87dcf9620724c in 1291ms, sequenceid=323, compaction requested=true 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:47,186 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:47,186 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:47,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:47,186 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:47,186 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:47,187 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:47,187 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:47,187 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/77df9e474a5c459cac9999f7fb10a3e3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/512a7b518d1141a79e45908a679afcc1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/ce3e781a73ad4291928a280cdd0ca87c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.7 K 2024-12-09T19:05:47,187 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/51aa484316a64732a44e20d1d5092cad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/801c57d2f5474625933f46b5ce796c48, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e8ff1734a66a473aba73f4e305bc4f34] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=41.5 K 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 77df9e474a5c459cac9999f7fb10a3e3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733771142551 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51aa484316a64732a44e20d1d5092cad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733771142551 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 512a7b518d1141a79e45908a679afcc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733771143718 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 801c57d2f5474625933f46b5ce796c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733771143718 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3e781a73ad4291928a280cdd0ca87c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771145890 2024-12-09T19:05:47,187 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8ff1734a66a473aba73f4e305bc4f34, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771145889 2024-12-09T19:05:47,194 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:47,194 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1a23e16fb0c94d5583b8cdcb67ac1547 is 50, key is test_row_0/B:col10/1733771145893/Put/seqid=0 2024-12-09T19:05:47,197 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:47,197 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/fa6674492e5a4a169f15a0d1ae072504 is 50, key is test_row_0/A:col10/1733771145893/Put/seqid=0 2024-12-09T19:05:47,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742259_1435 (size=13085) 2024-12-09T19:05:47,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742260_1436 (size=13085) 2024-12-09T19:05:47,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T19:05:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:47,324 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:05:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/0adcfe02e0884bef92c7819398fd73db is 50, key is test_row_0/A:col10/1733771145930/Put/seqid=0 2024-12-09T19:05:47,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742261_1437 (size=12301) 2024-12-09T19:05:47,604 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1a23e16fb0c94d5583b8cdcb67ac1547 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1a23e16fb0c94d5583b8cdcb67ac1547 2024-12-09T19:05:47,607 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 1a23e16fb0c94d5583b8cdcb67ac1547(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:47,607 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:47,607 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=13, startTime=1733771147186; duration=0sec 2024-12-09T19:05:47,607 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:47,607 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:47,608 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:47,608 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:47,608 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:47,608 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:47,608 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ec5d5bb657be4007bc8815f5c0c32c45, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/208e7a71e2f346a1aa5d5a3045e74000] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.7 K 2024-12-09T19:05:47,609 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5d5bb657be4007bc8815f5c0c32c45, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733771142551 2024-12-09T19:05:47,609 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd2b7ec2c154f96bfa99b5c6bb5adf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733771143718 2024-12-09T19:05:47,609 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 208e7a71e2f346a1aa5d5a3045e74000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771145890 2024-12-09T19:05:47,611 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/fa6674492e5a4a169f15a0d1ae072504 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/fa6674492e5a4a169f15a0d1ae072504 2024-12-09T19:05:47,615 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:47,615 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into fa6674492e5a4a169f15a0d1ae072504(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:47,615 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:47,615 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=13, startTime=1733771147186; duration=0sec 2024-12-09T19:05:47,615 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:47,615 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/d73a14deeb704fc88fb79a9cec4db1dd is 50, key is test_row_0/C:col10/1733771145893/Put/seqid=0 2024-12-09T19:05:47,615 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742262_1438 (size=13085) 2024-12-09T19:05:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T19:05:47,733 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/0adcfe02e0884bef92c7819398fd73db 2024-12-09T19:05:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/a8a9adf723e9410b94910d3679c071e1 is 50, key is test_row_0/B:col10/1733771145930/Put/seqid=0 2024-12-09T19:05:47,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742263_1439 (size=12301) 2024-12-09T19:05:47,742 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/a8a9adf723e9410b94910d3679c071e1 2024-12-09T19:05:47,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/9c4211d6304d433fa25ad54c5939eae6 is 50, key is test_row_0/C:col10/1733771145930/Put/seqid=0 2024-12-09T19:05:47,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742264_1440 (size=12301) 2024-12-09T19:05:48,025 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/d73a14deeb704fc88fb79a9cec4db1dd as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/d73a14deeb704fc88fb79a9cec4db1dd 2024-12-09T19:05:48,029 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into d73a14deeb704fc88fb79a9cec4db1dd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:48,029 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:48,029 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=13, startTime=1733771147186; duration=0sec 2024-12-09T19:05:48,029 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:48,029 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:48,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. as already flushing 2024-12-09T19:05:48,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771208127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771208128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771208129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771208130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771208131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,151 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/9c4211d6304d433fa25ad54c5939eae6 2024-12-09T19:05:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/0adcfe02e0884bef92c7819398fd73db as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/0adcfe02e0884bef92c7819398fd73db 2024-12-09T19:05:48,158 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/0adcfe02e0884bef92c7819398fd73db, entries=150, sequenceid=347, filesize=12.0 K 2024-12-09T19:05:48,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/a8a9adf723e9410b94910d3679c071e1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8a9adf723e9410b94910d3679c071e1 2024-12-09T19:05:48,162 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8a9adf723e9410b94910d3679c071e1, entries=150, sequenceid=347, filesize=12.0 K 2024-12-09T19:05:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/9c4211d6304d433fa25ad54c5939eae6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9c4211d6304d433fa25ad54c5939eae6 2024-12-09T19:05:48,166 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9c4211d6304d433fa25ad54c5939eae6, entries=150, sequenceid=347, filesize=12.0 K 2024-12-09T19:05:48,167 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e7537c2a1583e00ca6f87dcf9620724c in 843ms, sequenceid=347, compaction requested=false 2024-12-09T19:05:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:48,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T19:05:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T19:05:48,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T19:05:48,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-12-09T19:05:48,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6450 sec 2024-12-09T19:05:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:48,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:48,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:05:48,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/390fa118fc334772b822e7e57068d393 is 50, key is test_row_0/A:col10/1733771148237/Put/seqid=0 2024-12-09T19:05:48,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742265_1441 (size=14741) 2024-12-09T19:05:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771208273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771208274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771208274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771208281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771208282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771208385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771208385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771208394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771208394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771208394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,441 INFO [Thread-1697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1697. 2024-12-09T19:05:48,441 INFO [Thread-1695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1695. 2024-12-09T19:05:48,441 DEBUG [Thread-1697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:48,441 DEBUG [Thread-1695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:48,441 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,441 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,441 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,441 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:48,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:48,443 INFO [Thread-1691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1691. 2024-12-09T19:05:48,443 DEBUG [Thread-1691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:48,443 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,443 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:48,447 INFO [Thread-1689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1689. 2024-12-09T19:05:48,448 DEBUG [Thread-1689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:48,448 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,448 INFO [Thread-1693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1693. 2024-12-09T19:05:48,448 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,448 DEBUG [Thread-1693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:48,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:48,448 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,448 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:48,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771208595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771208595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771208605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771208605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771208605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/390fa118fc334772b822e7e57068d393 2024-12-09T19:05:48,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1fb256ac3aa14886a119a4e1ebc0da3f is 50, key is test_row_0/B:col10/1733771148237/Put/seqid=0 2024-12-09T19:05:48,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742266_1442 (size=12301) 2024-12-09T19:05:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T19:05:48,675 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771208905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771208906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771208915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771208915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771208915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:49,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1fb256ac3aa14886a119a4e1ebc0da3f 2024-12-09T19:05:49,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb is 50, key is test_row_0/C:col10/1733771148237/Put/seqid=0 2024-12-09T19:05:49,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742267_1443 (size=12301) 2024-12-09T19:05:49,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:49,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37320 deadline: 1733771209415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37308 deadline: 1733771209415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1733771209426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37262 deadline: 1733771209426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37266 deadline: 1733771209426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 , the old value is region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=2 because the exception is null or not the one we care about 2024-12-09T19:05:49,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb 2024-12-09T19:05:49,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/390fa118fc334772b822e7e57068d393 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/390fa118fc334772b822e7e57068d393 2024-12-09T19:05:49,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/390fa118fc334772b822e7e57068d393, entries=200, sequenceid=363, filesize=14.4 K 2024-12-09T19:05:49,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1fb256ac3aa14886a119a4e1ebc0da3f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb256ac3aa14886a119a4e1ebc0da3f 2024-12-09T19:05:49,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb256ac3aa14886a119a4e1ebc0da3f, entries=150, sequenceid=363, filesize=12.0 K 2024-12-09T19:05:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb 2024-12-09T19:05:49,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb, entries=150, sequenceid=363, filesize=12.0 K 2024-12-09T19:05:49,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e7537c2a1583e00ca6f87dcf9620724c in 1266ms, sequenceid=363, compaction requested=true 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:49,505 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7537c2a1583e00ca6f87dcf9620724c:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:05:49,505 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/A is initiating minor compaction (all files) 2024-12-09T19:05:49,506 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/A in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/B is initiating minor compaction (all files) 2024-12-09T19:05:49,506 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/B in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:49,506 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/fa6674492e5a4a169f15a0d1ae072504, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/0adcfe02e0884bef92c7819398fd73db, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/390fa118fc334772b822e7e57068d393] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=39.2 K 2024-12-09T19:05:49,506 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1a23e16fb0c94d5583b8cdcb67ac1547, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8a9adf723e9410b94910d3679c071e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb256ac3aa14886a119a4e1ebc0da3f] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.8 K 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa6674492e5a4a169f15a0d1ae072504, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771145890 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a23e16fb0c94d5583b8cdcb67ac1547, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771145890 2024-12-09T19:05:49,506 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0adcfe02e0884bef92c7819398fd73db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733771145927 2024-12-09T19:05:49,507 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a9adf723e9410b94910d3679c071e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733771145927 2024-12-09T19:05:49,507 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 390fa118fc334772b822e7e57068d393, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733771148126 2024-12-09T19:05:49,507 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb256ac3aa14886a119a4e1ebc0da3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733771148126 2024-12-09T19:05:49,511 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#A#compaction#381 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:49,512 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/c84d6e8d73e7428d808e1e7f4af16eba is 50, key is test_row_0/A:col10/1733771148237/Put/seqid=0 2024-12-09T19:05:49,512 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:49,512 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1fb12b82a3c14fefb3326e293bd0df95 is 50, key is test_row_0/B:col10/1733771148237/Put/seqid=0 2024-12-09T19:05:49,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742268_1444 (size=13187) 2024-12-09T19:05:49,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742269_1445 (size=13187) 2024-12-09T19:05:49,925 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/1fb12b82a3c14fefb3326e293bd0df95 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb12b82a3c14fefb3326e293bd0df95 2024-12-09T19:05:49,925 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/c84d6e8d73e7428d808e1e7f4af16eba as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/c84d6e8d73e7428d808e1e7f4af16eba 2024-12-09T19:05:49,932 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/B of e7537c2a1583e00ca6f87dcf9620724c into 1fb12b82a3c14fefb3326e293bd0df95(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:49,932 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/A of e7537c2a1583e00ca6f87dcf9620724c into c84d6e8d73e7428d808e1e7f4af16eba(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:49,932 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/A, priority=13, startTime=1733771149505; duration=0sec 2024-12-09T19:05:49,932 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/B, priority=13, startTime=1733771149505; duration=0sec 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:B 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:A 2024-12-09T19:05:49,932 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:05:49,933 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:05:49,934 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): e7537c2a1583e00ca6f87dcf9620724c/C is initiating minor compaction (all files) 2024-12-09T19:05:49,934 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7537c2a1583e00ca6f87dcf9620724c/C in TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:49,934 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/d73a14deeb704fc88fb79a9cec4db1dd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9c4211d6304d433fa25ad54c5939eae6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp, totalSize=36.8 K 2024-12-09T19:05:49,934 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting d73a14deeb704fc88fb79a9cec4db1dd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733771145890 2024-12-09T19:05:49,934 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4211d6304d433fa25ad54c5939eae6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733771145927 2024-12-09T19:05:49,935 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf0d5caa4ab44e8b80d4ef3c1d015cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733771148126 2024-12-09T19:05:49,941 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7537c2a1583e00ca6f87dcf9620724c#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:05:49,942 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/812db7fa489f4aada724e4df09319a3d is 50, key is test_row_0/C:col10/1733771148237/Put/seqid=0 2024-12-09T19:05:49,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742270_1446 (size=13187) 2024-12-09T19:05:50,354 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/812db7fa489f4aada724e4df09319a3d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/812db7fa489f4aada724e4df09319a3d 2024-12-09T19:05:50,358 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7537c2a1583e00ca6f87dcf9620724c/C of e7537c2a1583e00ca6f87dcf9620724c into 812db7fa489f4aada724e4df09319a3d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:05:50,358 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:50,358 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., storeName=e7537c2a1583e00ca6f87dcf9620724c/C, priority=13, startTime=1733771149505; duration=0sec 2024-12-09T19:05:50,358 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:05:50,359 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7537c2a1583e00ca6f87dcf9620724c:C 2024-12-09T19:05:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:50,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:05:50,428 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-12-09T19:05:50,428 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:50,428 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,429 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:50,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/3edc03d5151d4dd492ee01681f5bb3b3 is 50, key is test_row_0/A:col10/1733771150425/Put/seqid=0 2024-12-09T19:05:50,437 INFO [Thread-1686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1686. 2024-12-09T19:05:50,437 INFO [Thread-1684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1684. 2024-12-09T19:05:50,437 INFO [Thread-1678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1678. 2024-12-09T19:05:50,437 DEBUG [Thread-1686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:50,437 DEBUG [Thread-1684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:50,437 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,437 DEBUG [Thread-1678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:50,437 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,437 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,437 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,437 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:50,437 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:50,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:50,438 INFO [Thread-1682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1682. 2024-12-09T19:05:50,438 DEBUG [Thread-1682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:05:50,439 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,439 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1978 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5925 rows 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1978 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5932 rows 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1976 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5926 rows 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1966 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5896 rows 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1984 2024-12-09T19:05:50,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5952 rows 2024-12-09T19:05:50,439 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:05:50,439 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:05:50,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:50,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:05:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742271_1447 (size=12301) 2024-12-09T19:05:50,440 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T19:05:50,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/3edc03d5151d4dd492ee01681f5bb3b3 2024-12-09T19:05:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:50,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771150445"}]},"ts":"1733771150445"} 2024-12-09T19:05:50,447 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T19:05:50,447 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T19:05:50,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:05:50,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/b2aff7af20ba4257a117b425ea534609 is 50, key is test_row_0/B:col10/1733771150425/Put/seqid=0 2024-12-09T19:05:50,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, UNASSIGN}] 2024-12-09T19:05:50,450 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, UNASSIGN 2024-12-09T19:05:50,451 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=e7537c2a1583e00ca6f87dcf9620724c, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:50,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742272_1448 (size=12301) 2024-12-09T19:05:50,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, UNASSIGN because future has completed 2024-12-09T19:05:50,453 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:05:50,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:05:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:50,606 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:50,606 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:05:50,606 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing e7537c2a1583e00ca6f87dcf9620724c, disabling compactions & flushes 2024-12-09T19:05:50,606 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:50,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/b2aff7af20ba4257a117b425ea534609 2024-12-09T19:05:50,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/e57b3c7d9dea4426ac7d0028fb7ab688 is 50, key is test_row_0/C:col10/1733771150425/Put/seqid=0 2024-12-09T19:05:50,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742273_1449 (size=12301) 2024-12-09T19:05:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:51,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/e57b3c7d9dea4426ac7d0028fb7ab688 2024-12-09T19:05:51,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/3edc03d5151d4dd492ee01681f5bb3b3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3edc03d5151d4dd492ee01681f5bb3b3 2024-12-09T19:05:51,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3edc03d5151d4dd492ee01681f5bb3b3, entries=150, sequenceid=390, filesize=12.0 K 2024-12-09T19:05:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/b2aff7af20ba4257a117b425ea534609 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b2aff7af20ba4257a117b425ea534609 2024-12-09T19:05:51,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b2aff7af20ba4257a117b425ea534609, entries=150, sequenceid=390, filesize=12.0 K 2024-12-09T19:05:51,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/e57b3c7d9dea4426ac7d0028fb7ab688 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/e57b3c7d9dea4426ac7d0028fb7ab688 2024-12-09T19:05:51,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/e57b3c7d9dea4426ac7d0028fb7ab688, entries=150, sequenceid=390, filesize=12.0 K 2024-12-09T19:05:51,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for e7537c2a1583e00ca6f87dcf9620724c in 866ms, sequenceid=390, compaction requested=false 2024-12-09T19:05:51,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7537c2a1583e00ca6f87dcf9620724c: 2024-12-09T19:05:51,295 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:51,295 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:51,295 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. after waiting 0 ms 2024-12-09T19:05:51,295 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:51,295 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing e7537c2a1583e00ca6f87dcf9620724c 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-09T19:05:51,299 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b19347bc67bd4d3a98838123db0e46a1 is 50, key is test_row_0/A:col10/1733771150435/Put/seqid=0 2024-12-09T19:05:51,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742274_1450 (size=9857) 2024-12-09T19:05:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:51,704 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b19347bc67bd4d3a98838123db0e46a1 2024-12-09T19:05:51,716 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/9a29c7f8e8b9463e9e442e187fed7aef is 50, key is test_row_0/B:col10/1733771150435/Put/seqid=0 2024-12-09T19:05:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742275_1451 (size=9857) 2024-12-09T19:05:52,121 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/9a29c7f8e8b9463e9e442e187fed7aef 2024-12-09T19:05:52,166 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/f2b9bc759df3456ead6744a04843a5cb is 50, key is test_row_0/C:col10/1733771150435/Put/seqid=0 2024-12-09T19:05:52,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742276_1452 (size=9857) 2024-12-09T19:05:52,573 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/f2b9bc759df3456ead6744a04843a5cb 2024-12-09T19:05:52,583 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/A/b19347bc67bd4d3a98838123db0e46a1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b19347bc67bd4d3a98838123db0e46a1 2024-12-09T19:05:52,588 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b19347bc67bd4d3a98838123db0e46a1, entries=100, sequenceid=397, filesize=9.6 K 2024-12-09T19:05:52,589 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/B/9a29c7f8e8b9463e9e442e187fed7aef as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/9a29c7f8e8b9463e9e442e187fed7aef 2024-12-09T19:05:52,594 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/9a29c7f8e8b9463e9e442e187fed7aef, entries=100, sequenceid=397, filesize=9.6 K 2024-12-09T19:05:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:52,595 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/.tmp/C/f2b9bc759df3456ead6744a04843a5cb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f2b9bc759df3456ead6744a04843a5cb 2024-12-09T19:05:52,599 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f2b9bc759df3456ead6744a04843a5cb, entries=100, sequenceid=397, filesize=9.6 K 2024-12-09T19:05:52,600 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e7537c2a1583e00ca6f87dcf9620724c in 1305ms, sequenceid=397, compaction requested=true 2024-12-09T19:05:52,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/5b9930c9b7494ddd9446b1fa54a1e69c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/23c8f9dc324d4ee2a17cf82381001b50, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/289e5befef414081bb73ed2cd65e0e77, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b23954f90a1f48629e64152990e12626, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/1e274fdb4a6b4658bba651c766e650d4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/325c322230b141dbb5e472588f59e897, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3e0540427354464b9d00697001ded3b8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e7e614070cd54dd98dbffbdb48af2011, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ba7512ca6cb8457aa7a8ef0c523aac90, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7f2bd9c4798e44638eab4490d7727b70, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b97cd4ed71a24108955791d5ac20121e, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b9787c9db2ef408cb1065cb797793c63, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/da38bf57641640f9bf5ed3a0663cf066, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ac774bd5e73340d79fe4a2402deab2bc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ef1e78e5cbc348b99137ee5584d9d7e0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e19cc4b9354a4a8bba460cbc27e82c8f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/40544b5c201f4d3c9e87fe5ab0711d0d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e60dfc3fd2ad4b2fbbc95e27177e60f5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7bcb174f908743f180b63f8a37940cc5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/88a49f47bba24d959af74832ab4545b1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/51aa484316a64732a44e20d1d5092cad, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/801c57d2f5474625933f46b5ce796c48, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e8ff1734a66a473aba73f4e305bc4f34, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/fa6674492e5a4a169f15a0d1ae072504, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/0adcfe02e0884bef92c7819398fd73db, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/390fa118fc334772b822e7e57068d393] to archive 2024-12-09T19:05:52,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:52,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/5b9930c9b7494ddd9446b1fa54a1e69c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/5b9930c9b7494ddd9446b1fa54a1e69c 2024-12-09T19:05:52,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/23c8f9dc324d4ee2a17cf82381001b50 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/23c8f9dc324d4ee2a17cf82381001b50 2024-12-09T19:05:52,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/289e5befef414081bb73ed2cd65e0e77 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/289e5befef414081bb73ed2cd65e0e77 2024-12-09T19:05:52,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b23954f90a1f48629e64152990e12626 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b23954f90a1f48629e64152990e12626 2024-12-09T19:05:52,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/1e274fdb4a6b4658bba651c766e650d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/1e274fdb4a6b4658bba651c766e650d4 2024-12-09T19:05:52,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/325c322230b141dbb5e472588f59e897 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/325c322230b141dbb5e472588f59e897 2024-12-09T19:05:52,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3e0540427354464b9d00697001ded3b8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3e0540427354464b9d00697001ded3b8 2024-12-09T19:05:52,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e7e614070cd54dd98dbffbdb48af2011 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e7e614070cd54dd98dbffbdb48af2011 2024-12-09T19:05:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ba7512ca6cb8457aa7a8ef0c523aac90 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ba7512ca6cb8457aa7a8ef0c523aac90 2024-12-09T19:05:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7f2bd9c4798e44638eab4490d7727b70 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7f2bd9c4798e44638eab4490d7727b70 2024-12-09T19:05:52,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b97cd4ed71a24108955791d5ac20121e to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b97cd4ed71a24108955791d5ac20121e 2024-12-09T19:05:52,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b9787c9db2ef408cb1065cb797793c63 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b9787c9db2ef408cb1065cb797793c63 2024-12-09T19:05:52,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/da38bf57641640f9bf5ed3a0663cf066 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/da38bf57641640f9bf5ed3a0663cf066 2024-12-09T19:05:52,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ac774bd5e73340d79fe4a2402deab2bc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ac774bd5e73340d79fe4a2402deab2bc 2024-12-09T19:05:52,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ef1e78e5cbc348b99137ee5584d9d7e0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/ef1e78e5cbc348b99137ee5584d9d7e0 2024-12-09T19:05:52,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e19cc4b9354a4a8bba460cbc27e82c8f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e19cc4b9354a4a8bba460cbc27e82c8f 2024-12-09T19:05:52,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/40544b5c201f4d3c9e87fe5ab0711d0d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/40544b5c201f4d3c9e87fe5ab0711d0d 2024-12-09T19:05:52,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e60dfc3fd2ad4b2fbbc95e27177e60f5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e60dfc3fd2ad4b2fbbc95e27177e60f5 2024-12-09T19:05:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7bcb174f908743f180b63f8a37940cc5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/7bcb174f908743f180b63f8a37940cc5 2024-12-09T19:05:52,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/88a49f47bba24d959af74832ab4545b1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/88a49f47bba24d959af74832ab4545b1 2024-12-09T19:05:52,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/51aa484316a64732a44e20d1d5092cad to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/51aa484316a64732a44e20d1d5092cad 2024-12-09T19:05:52,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/801c57d2f5474625933f46b5ce796c48 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/801c57d2f5474625933f46b5ce796c48 2024-12-09T19:05:52,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e8ff1734a66a473aba73f4e305bc4f34 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/e8ff1734a66a473aba73f4e305bc4f34 2024-12-09T19:05:52,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/fa6674492e5a4a169f15a0d1ae072504 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/fa6674492e5a4a169f15a0d1ae072504 2024-12-09T19:05:52,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/0adcfe02e0884bef92c7819398fd73db to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/0adcfe02e0884bef92c7819398fd73db 2024-12-09T19:05:52,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/390fa118fc334772b822e7e57068d393 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/390fa118fc334772b822e7e57068d393 2024-12-09T19:05:52,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/4fed995bcd9f4818b457c146c5ce7711, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/8084f8e3260f417786776255822f8990, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/2bd2c6fd145546cdb79e72fdc93a2f54, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/74d1dca6ba6c429fb7794ba75a1312b0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5c6fc32fc73b4a31ac3b360a0062b953, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/0d2215c5862f4a23b56d0e0d56371268, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8dc94214f3149d4902365fd1f909319, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/88a041a250154d58a806f43052b9d17a, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b64b47a941e04c2d9866bbaeff9aa18b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/bd07acfebf6c454098bfa15f9dca1428, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e6af3e229d9f4701ae219a3b9908a715, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/c7e78f0256774d21ba7d053a4f0280af, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5b07600e1e794f6e85c99963c799befe, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1e58014881fe449ca0fef1ae0a11a642, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/62968ad22622447f915773e63e4ede27, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/34707b17a81242148dbb82eaed079d56, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/04632d092d7d462b8b4ee53d4f9c2f2f, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/993d24b239b2449e8152d09e32fbfc4b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/77df9e474a5c459cac9999f7fb10a3e3, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e184a230255a4d00ba0b188766b2f768, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/512a7b518d1141a79e45908a679afcc1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1a23e16fb0c94d5583b8cdcb67ac1547, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/ce3e781a73ad4291928a280cdd0ca87c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8a9adf723e9410b94910d3679c071e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb256ac3aa14886a119a4e1ebc0da3f] to archive 2024-12-09T19:05:52,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:52,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/4fed995bcd9f4818b457c146c5ce7711 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/4fed995bcd9f4818b457c146c5ce7711 2024-12-09T19:05:52,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5f0f90ac5e7e4ae2af1f3aab89f7a6d5 2024-12-09T19:05:52,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/8084f8e3260f417786776255822f8990 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/8084f8e3260f417786776255822f8990 2024-12-09T19:05:52,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/2bd2c6fd145546cdb79e72fdc93a2f54 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/2bd2c6fd145546cdb79e72fdc93a2f54 2024-12-09T19:05:52,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/74d1dca6ba6c429fb7794ba75a1312b0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/74d1dca6ba6c429fb7794ba75a1312b0 2024-12-09T19:05:52,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5c6fc32fc73b4a31ac3b360a0062b953 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5c6fc32fc73b4a31ac3b360a0062b953 2024-12-09T19:05:52,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/0d2215c5862f4a23b56d0e0d56371268 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/0d2215c5862f4a23b56d0e0d56371268 2024-12-09T19:05:52,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8dc94214f3149d4902365fd1f909319 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8dc94214f3149d4902365fd1f909319 2024-12-09T19:05:52,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/88a041a250154d58a806f43052b9d17a to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/88a041a250154d58a806f43052b9d17a 2024-12-09T19:05:52,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b64b47a941e04c2d9866bbaeff9aa18b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b64b47a941e04c2d9866bbaeff9aa18b 2024-12-09T19:05:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/bd07acfebf6c454098bfa15f9dca1428 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/bd07acfebf6c454098bfa15f9dca1428 2024-12-09T19:05:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e6af3e229d9f4701ae219a3b9908a715 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e6af3e229d9f4701ae219a3b9908a715 2024-12-09T19:05:52,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/c7e78f0256774d21ba7d053a4f0280af to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/c7e78f0256774d21ba7d053a4f0280af 2024-12-09T19:05:52,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5b07600e1e794f6e85c99963c799befe to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/5b07600e1e794f6e85c99963c799befe 2024-12-09T19:05:52,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1e58014881fe449ca0fef1ae0a11a642 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1e58014881fe449ca0fef1ae0a11a642 2024-12-09T19:05:52,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/62968ad22622447f915773e63e4ede27 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/62968ad22622447f915773e63e4ede27 2024-12-09T19:05:52,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/34707b17a81242148dbb82eaed079d56 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/34707b17a81242148dbb82eaed079d56 2024-12-09T19:05:52,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/04632d092d7d462b8b4ee53d4f9c2f2f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/04632d092d7d462b8b4ee53d4f9c2f2f 2024-12-09T19:05:52,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/993d24b239b2449e8152d09e32fbfc4b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/993d24b239b2449e8152d09e32fbfc4b 2024-12-09T19:05:52,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/77df9e474a5c459cac9999f7fb10a3e3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/77df9e474a5c459cac9999f7fb10a3e3 2024-12-09T19:05:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e184a230255a4d00ba0b188766b2f768 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/e184a230255a4d00ba0b188766b2f768 2024-12-09T19:05:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/512a7b518d1141a79e45908a679afcc1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/512a7b518d1141a79e45908a679afcc1 2024-12-09T19:05:52,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1a23e16fb0c94d5583b8cdcb67ac1547 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1a23e16fb0c94d5583b8cdcb67ac1547 2024-12-09T19:05:52,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/ce3e781a73ad4291928a280cdd0ca87c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/ce3e781a73ad4291928a280cdd0ca87c 2024-12-09T19:05:52,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8a9adf723e9410b94910d3679c071e1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/a8a9adf723e9410b94910d3679c071e1 2024-12-09T19:05:52,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb256ac3aa14886a119a4e1ebc0da3f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb256ac3aa14886a119a4e1ebc0da3f 2024-12-09T19:05:52,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/94b32b1cf17a44fbbbcdb46c3a5d1237, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/8b0bad5f6c394ae7addf898bd2e82e2c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/78a57c6b0e5048a0b19720b4314cb8bb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/be5eda9fde054c72b22f7bb6a892bd48, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/2b1cb78029eb4c48a2bff2919f3fda5c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/15a3733f9aa442b19a328b54dee2765d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/49cd15d573744f15b2b5e7f65e014c0b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/44196552d0b44e29aee0d0baea1ca913, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eedcdc298477425491cf74848aef56dc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9059a872456246808fe45b5202e47f36, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/1ee48f360a9f4b57b3479e83cbb5d4fc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/87f6b416ddc7459abd3fb47f1449c84c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/b03fa62bc470466191fba47f24b39fd0, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c3687973fc32437ea5c23e0ec4627358, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eae17031f8634342b80dd8ae797774a4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/544e9ec437ea46f0892edb4e4bcce0f1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/11f4819cdc9041f9b0c4c6d9ac9d4efc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f356109a60a64cbebd9ec74c247339a4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/55fa81ab44d040fba7d76e5084a2d8fa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ec5d5bb657be4007bc8815f5c0c32c45, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c2fd2e5edfd9469daa81750c1a5dc1b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/d73a14deeb704fc88fb79a9cec4db1dd, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/208e7a71e2f346a1aa5d5a3045e74000, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9c4211d6304d433fa25ad54c5939eae6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb] to archive 2024-12-09T19:05:52,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:05:52,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/94b32b1cf17a44fbbbcdb46c3a5d1237 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/94b32b1cf17a44fbbbcdb46c3a5d1237 2024-12-09T19:05:52,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/8b0bad5f6c394ae7addf898bd2e82e2c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/8b0bad5f6c394ae7addf898bd2e82e2c 2024-12-09T19:05:52,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/78a57c6b0e5048a0b19720b4314cb8bb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/78a57c6b0e5048a0b19720b4314cb8bb 2024-12-09T19:05:52,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/be5eda9fde054c72b22f7bb6a892bd48 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/be5eda9fde054c72b22f7bb6a892bd48 2024-12-09T19:05:52,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/2b1cb78029eb4c48a2bff2919f3fda5c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/2b1cb78029eb4c48a2bff2919f3fda5c 2024-12-09T19:05:52,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/15a3733f9aa442b19a328b54dee2765d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/15a3733f9aa442b19a328b54dee2765d 2024-12-09T19:05:52,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/49cd15d573744f15b2b5e7f65e014c0b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/49cd15d573744f15b2b5e7f65e014c0b 2024-12-09T19:05:52,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/44196552d0b44e29aee0d0baea1ca913 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/44196552d0b44e29aee0d0baea1ca913 2024-12-09T19:05:52,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eedcdc298477425491cf74848aef56dc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eedcdc298477425491cf74848aef56dc 2024-12-09T19:05:52,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9059a872456246808fe45b5202e47f36 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9059a872456246808fe45b5202e47f36 2024-12-09T19:05:52,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/1ee48f360a9f4b57b3479e83cbb5d4fc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/1ee48f360a9f4b57b3479e83cbb5d4fc 2024-12-09T19:05:52,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/87f6b416ddc7459abd3fb47f1449c84c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/87f6b416ddc7459abd3fb47f1449c84c 2024-12-09T19:05:52,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/b03fa62bc470466191fba47f24b39fd0 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/b03fa62bc470466191fba47f24b39fd0 2024-12-09T19:05:52,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c3687973fc32437ea5c23e0ec4627358 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c3687973fc32437ea5c23e0ec4627358 2024-12-09T19:05:52,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eae17031f8634342b80dd8ae797774a4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/eae17031f8634342b80dd8ae797774a4 2024-12-09T19:05:52,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/544e9ec437ea46f0892edb4e4bcce0f1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/544e9ec437ea46f0892edb4e4bcce0f1 2024-12-09T19:05:52,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/11f4819cdc9041f9b0c4c6d9ac9d4efc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/11f4819cdc9041f9b0c4c6d9ac9d4efc 2024-12-09T19:05:52,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f356109a60a64cbebd9ec74c247339a4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f356109a60a64cbebd9ec74c247339a4 2024-12-09T19:05:52,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/55fa81ab44d040fba7d76e5084a2d8fa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/55fa81ab44d040fba7d76e5084a2d8fa 2024-12-09T19:05:52,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ec5d5bb657be4007bc8815f5c0c32c45 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ec5d5bb657be4007bc8815f5c0c32c45 2024-12-09T19:05:52,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c2fd2e5edfd9469daa81750c1a5dc1b4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/c2fd2e5edfd9469daa81750c1a5dc1b4 2024-12-09T19:05:52,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/0cd2b7ec2c154f96bfa99b5c6bb5adf5 2024-12-09T19:05:52,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/d73a14deeb704fc88fb79a9cec4db1dd to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/d73a14deeb704fc88fb79a9cec4db1dd 2024-12-09T19:05:52,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/208e7a71e2f346a1aa5d5a3045e74000 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/208e7a71e2f346a1aa5d5a3045e74000 2024-12-09T19:05:52,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9c4211d6304d433fa25ad54c5939eae6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/9c4211d6304d433fa25ad54c5939eae6 2024-12-09T19:05:52,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/ddf0d5caa4ab44e8b80d4ef3c1d015cb 2024-12-09T19:05:52,672 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/recovered.edits/400.seqid, newMaxSeqId=400, maxSeqId=1 2024-12-09T19:05:52,672 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. 2024-12-09T19:05:52,672 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for e7537c2a1583e00ca6f87dcf9620724c: Waiting for close lock at 1733771150606Running coprocessor pre-close hooks at 1733771150606Disabling compacts and flushes for region at 1733771150606Disabling writes for close at 1733771151295 (+689 ms)Obtaining lock to block concurrent updates at 1733771151295Preparing flush snapshotting stores in e7537c2a1583e00ca6f87dcf9620724c at 1733771151295Finished memstore snapshotting TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733771151295Flushing stores of TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c. at 1733771151296 (+1 ms)Flushing e7537c2a1583e00ca6f87dcf9620724c/A: creating writer at 1733771151296Flushing e7537c2a1583e00ca6f87dcf9620724c/A: appending metadata at 1733771151298 (+2 ms)Flushing e7537c2a1583e00ca6f87dcf9620724c/A: closing flushed file at 1733771151298Flushing e7537c2a1583e00ca6f87dcf9620724c/B: creating writer at 1733771151713 (+415 ms)Flushing e7537c2a1583e00ca6f87dcf9620724c/B: appending metadata at 1733771151715 (+2 ms)Flushing e7537c2a1583e00ca6f87dcf9620724c/B: closing flushed file at 1733771151715Flushing e7537c2a1583e00ca6f87dcf9620724c/C: creating writer at 1733771152161 (+446 ms)Flushing e7537c2a1583e00ca6f87dcf9620724c/C: appending metadata at 1733771152166 (+5 ms)Flushing e7537c2a1583e00ca6f87dcf9620724c/C: closing flushed file at 1733771152166Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c0ca395: reopening flushed file at 1733771152582 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59de6ffd: reopening flushed file at 1733771152588 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7124a175: reopening flushed file at 1733771152594 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e7537c2a1583e00ca6f87dcf9620724c in 1305ms, sequenceid=397, compaction requested=true at 1733771152600 (+6 ms)Writing region close event to WAL at 1733771152670 (+70 ms)Running coprocessor post-close hooks at 1733771152672 (+2 ms)Closed at 1733771152672 2024-12-09T19:05:52,673 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:52,674 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=e7537c2a1583e00ca6f87dcf9620724c, regionState=CLOSED 2024-12-09T19:05:52,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:05:52,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T19:05:52,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure e7537c2a1583e00ca6f87dcf9620724c, server=a437f8b9ba7d,46189,1733770998237 in 2.2230 sec 2024-12-09T19:05:52,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-12-09T19:05:52,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7537c2a1583e00ca6f87dcf9620724c, UNASSIGN in 2.2280 sec 2024-12-09T19:05:52,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T19:05:52,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2310 sec 2024-12-09T19:05:52,681 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771152681"}]},"ts":"1733771152681"} 2024-12-09T19:05:52,682 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T19:05:52,682 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T19:05:52,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2420 sec 2024-12-09T19:05:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T19:05:54,616 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:54,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T19:05:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,622 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T19:05:54,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,627 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:54,629 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/recovered.edits] 2024-12-09T19:05:54,631 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3edc03d5151d4dd492ee01681f5bb3b3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/3edc03d5151d4dd492ee01681f5bb3b3 2024-12-09T19:05:54,631 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b19347bc67bd4d3a98838123db0e46a1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/b19347bc67bd4d3a98838123db0e46a1 2024-12-09T19:05:54,632 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/c84d6e8d73e7428d808e1e7f4af16eba to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/A/c84d6e8d73e7428d808e1e7f4af16eba 2024-12-09T19:05:54,633 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb12b82a3c14fefb3326e293bd0df95 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/1fb12b82a3c14fefb3326e293bd0df95 2024-12-09T19:05:54,634 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/9a29c7f8e8b9463e9e442e187fed7aef to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/9a29c7f8e8b9463e9e442e187fed7aef 2024-12-09T19:05:54,635 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b2aff7af20ba4257a117b425ea534609 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/B/b2aff7af20ba4257a117b425ea534609 2024-12-09T19:05:54,636 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/812db7fa489f4aada724e4df09319a3d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/812db7fa489f4aada724e4df09319a3d 2024-12-09T19:05:54,637 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/e57b3c7d9dea4426ac7d0028fb7ab688 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/e57b3c7d9dea4426ac7d0028fb7ab688 2024-12-09T19:05:54,637 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f2b9bc759df3456ead6744a04843a5cb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/C/f2b9bc759df3456ead6744a04843a5cb 2024-12-09T19:05:54,639 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/recovered.edits/400.seqid to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c/recovered.edits/400.seqid 2024-12-09T19:05:54,639 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/e7537c2a1583e00ca6f87dcf9620724c 2024-12-09T19:05:54,639 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T19:05:54,641 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,642 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T19:05:54,644 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T19:05:54,645 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,645 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T19:05:54,645 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733771154645"}]},"ts":"9223372036854775807"} 2024-12-09T19:05:54,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T19:05:54,646 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e7537c2a1583e00ca6f87dcf9620724c, NAME => 'TestAcidGuarantees,,1733771126220.e7537c2a1583e00ca6f87dcf9620724c.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T19:05:54,646 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T19:05:54,646 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733771154646"}]},"ts":"9223372036854775807"} 2024-12-09T19:05:54,648 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T19:05:54,648 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-09T19:05:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T19:05:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T19:05:54,735 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:54,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 239) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 308) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2148 (was 2169) 2024-12-09T19:05:54,750 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=2148 2024-12-09T19:05:54,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:05:54,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:05:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:54,753 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T19:05:54,753 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:54,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-12-09T19:05:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-09T19:05:54,754 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T19:05:54,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742277_1453 (size=959) 2024-12-09T19:05:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-09T19:05:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-09T19:05:55,165 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b 2024-12-09T19:05:55,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742278_1454 (size=53) 2024-12-09T19:05:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-09T19:05:55,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:05:55,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cfa5a71b9d5357947521a419cca3f2d4, disabling compactions & flushes 2024-12-09T19:05:55,576 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. after waiting 0 ms 2024-12-09T19:05:55,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,576 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,576 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfa5a71b9d5357947521a419cca3f2d4: Waiting for close lock at 1733771155576Disabling compacts and flushes for region at 1733771155576Disabling writes for close at 1733771155576Writing region close event to WAL at 1733771155576Closed at 1733771155576 2024-12-09T19:05:55,579 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T19:05:55,580 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733771155579"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733771155579"}]},"ts":"1733771155579"} 2024-12-09T19:05:55,583 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T19:05:55,584 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T19:05:55,584 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771155584"}]},"ts":"1733771155584"} 2024-12-09T19:05:55,586 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T19:05:55,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, ASSIGN}] 2024-12-09T19:05:55,588 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, ASSIGN 2024-12-09T19:05:55,589 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=false 2024-12-09T19:05:55,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:55,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, ASSIGN because future has completed 2024-12-09T19:05:55,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:05:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-09T19:05:55,907 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,907 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:05:55,907 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,907 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:05:55,907 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,907 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,909 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,911 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfa5a71b9d5357947521a419cca3f2d4 columnFamilyName A 2024-12-09T19:05:55,911 DEBUG [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:55,912 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(327): Store=cfa5a71b9d5357947521a419cca3f2d4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:55,912 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,913 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfa5a71b9d5357947521a419cca3f2d4 columnFamilyName B 2024-12-09T19:05:55,913 DEBUG [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:55,914 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(327): Store=cfa5a71b9d5357947521a419cca3f2d4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:55,914 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,915 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfa5a71b9d5357947521a419cca3f2d4 columnFamilyName C 2024-12-09T19:05:55,915 DEBUG [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:55,916 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(327): Store=cfa5a71b9d5357947521a419cca3f2d4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:55,916 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,916 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,917 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,918 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,918 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,919 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:05:55,920 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,922 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T19:05:55,923 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened cfa5a71b9d5357947521a419cca3f2d4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63957494, jitterRate=-0.04695907235145569}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:05:55,923 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:55,924 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for cfa5a71b9d5357947521a419cca3f2d4: Running coprocessor pre-open hook at 1733771155907Writing region info on filesystem at 1733771155907Initializing all the Stores at 1733771155908 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771155908Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771155909 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771155909Cleaning up temporary data from old regions at 1733771155918 (+9 ms)Running coprocessor post-open hooks at 1733771155923 (+5 ms)Region opened successfully at 1733771155924 (+1 ms) 2024-12-09T19:05:55,924 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., pid=132, masterSystemTime=1733771155899 2024-12-09T19:05:55,926 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,926 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:55,927 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:55,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:05:55,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T19:05:55,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 in 185 msec 2024-12-09T19:05:55,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-09T19:05:55,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, ASSIGN in 344 msec 2024-12-09T19:05:55,932 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T19:05:55,932 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771155932"}]},"ts":"1733771155932"} 2024-12-09T19:05:55,933 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T19:05:55,934 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T19:05:55,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-12-09T19:05:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-09T19:05:56,906 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:56,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:56,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:56,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:56,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e07d6f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:56,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6563dc1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:56,915 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:56,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:56,918 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T19:05:56,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,38053,1733770997561 2024-12-09T19:05:56,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48bcc92d 2024-12-09T19:05:56,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T19:05:56,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T19:05:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T19:05:56,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T19:05:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T19:05:56,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742279_1455 (size=995) 2024-12-09T19:05:57,390 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-09T19:05:57,390 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-09T19:05:57,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:05:57,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, REOPEN/MOVE}] 2024-12-09T19:05:57,401 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, REOPEN/MOVE 2024-12-09T19:05:57,401 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:57,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, REOPEN/MOVE because future has completed 2024-12-09T19:05:57,403 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:05:57,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:05:57,556 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,556 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:05:57,557 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing cfa5a71b9d5357947521a419cca3f2d4, disabling compactions & flushes 2024-12-09T19:05:57,557 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,557 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,557 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. after waiting 0 ms 2024-12-09T19:05:57,557 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,566 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T19:05:57,567 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,567 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for cfa5a71b9d5357947521a419cca3f2d4: Waiting for close lock at 1733771157557Running coprocessor pre-close hooks at 1733771157557Disabling compacts and flushes for region at 1733771157557Disabling writes for close at 1733771157557Writing region close event to WAL at 1733771157558 (+1 ms)Running coprocessor post-close hooks at 1733771157567 (+9 ms)Closed at 1733771157567 2024-12-09T19:05:57,568 WARN [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: cfa5a71b9d5357947521a419cca3f2d4 to self. 2024-12-09T19:05:57,571 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,572 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=CLOSED 2024-12-09T19:05:57,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:05:57,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T19:05:57,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 in 173 msec 2024-12-09T19:05:57,581 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, REOPEN/MOVE; state=CLOSED, location=a437f8b9ba7d,46189,1733770998237; forceNewPlan=false, retain=true 2024-12-09T19:05:57,731 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=OPENING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:57,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, REOPEN/MOVE because future has completed 2024-12-09T19:05:57,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:05:57,891 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,891 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} 2024-12-09T19:05:57,891 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,892 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T19:05:57,892 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,892 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,893 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,894 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfa5a71b9d5357947521a419cca3f2d4 columnFamilyName A 2024-12-09T19:05:57,896 DEBUG [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:57,897 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(327): Store=cfa5a71b9d5357947521a419cca3f2d4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:57,897 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,898 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfa5a71b9d5357947521a419cca3f2d4 columnFamilyName B 2024-12-09T19:05:57,898 DEBUG [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:57,899 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(327): Store=cfa5a71b9d5357947521a419cca3f2d4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:57,899 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,900 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfa5a71b9d5357947521a419cca3f2d4 columnFamilyName C 2024-12-09T19:05:57,900 DEBUG [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:57,901 INFO [StoreOpener-cfa5a71b9d5357947521a419cca3f2d4-1 {}] regionserver.HStore(327): Store=cfa5a71b9d5357947521a419cca3f2d4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T19:05:57,901 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,902 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,904 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,905 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,905 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,906 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T19:05:57,907 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,909 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened cfa5a71b9d5357947521a419cca3f2d4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71749511, jitterRate=0.06915102899074554}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T19:05:57,909 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,909 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for cfa5a71b9d5357947521a419cca3f2d4: Running coprocessor pre-open hook at 1733771157892Writing region info on filesystem at 1733771157892Initializing all the Stores at 1733771157893 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771157893Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771157893Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733771157893Cleaning up temporary data from old regions at 1733771157905 (+12 ms)Running coprocessor post-open hooks at 1733771157909 (+4 ms)Region opened successfully at 1733771157909 2024-12-09T19:05:57,910 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., pid=137, masterSystemTime=1733771157884 2024-12-09T19:05:57,912 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,912 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:57,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=OPEN, openSeqNum=5, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:57,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:05:57,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-12-09T19:05:57,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 in 182 msec 2024-12-09T19:05:57,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-09T19:05:57,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, REOPEN/MOVE in 517 msec 2024-12-09T19:05:57,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T19:05:57,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-12-09T19:05:57,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0000 sec 2024-12-09T19:05:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T19:05:57,925 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:05:57,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7518e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,928 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfc47f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ab88d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d3a7cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4575f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75f19139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a55c01b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635796ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a0f6608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c8f2ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d79c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a0233f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70811740, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3735f0df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,957 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6cdc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,959 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caa8d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4a46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,961 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38241459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,38053,-1 for getting cluster id 2024-12-09T19:05:57,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T19:05:57,963 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8165842d-7141-4c13-bea5-3a7be743de1e' 2024-12-09T19:05:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T19:05:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8165842d-7141-4c13-bea5-3a7be743de1e" 2024-12-09T19:05:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9bb31d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,38053,-1] 2024-12-09T19:05:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T19:05:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:05:57,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T19:05:57,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c1966e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T19:05:57,965 DEBUG [Thread-2073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [Thread-2084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [Thread-2075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [Thread-2080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,967 DEBUG [Thread-2086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,967 DEBUG [Thread-2078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,968 DEBUG [Thread-2082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T19:05:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:05:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46189,1733770998237, seqNum=-1] 2024-12-09T19:05:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T19:05:57,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T19:05:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-12-09T19:05:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:05:57,971 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:05:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,972 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:05:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:05:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5] 2024-12-09T19:05:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:57,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:05:57,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:05:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771217995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771217996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:57,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771217997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771217997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771217998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120979d6348652fb4a97955ffc98106214ce_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771157973/Put/seqid=0 2024-12-09T19:05:58,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742280_1456 (size=12154) 2024-12-09T19:05:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:05:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771218104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771218104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771218104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771218104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771218104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,192 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:05:58,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:05:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771218314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771218314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771218314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771218314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771218314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:58,412 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120979d6348652fb4a97955ffc98106214ce_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979d6348652fb4a97955ffc98106214ce_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:58,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/a93d9abd609e4c8a8294fe75a0667be2, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:05:58,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/a93d9abd609e4c8a8294fe75a0667be2 is 175, key is test_row_0/A:col10/1733771157973/Put/seqid=0 2024-12-09T19:05:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742281_1457 (size=30955) 2024-12-09T19:05:58,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:05:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771218624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771218624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771218624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771218624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771218624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:58,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/a93d9abd609e4c8a8294fe75a0667be2 2024-12-09T19:05:58,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/925daa12a73a4c458d5aa8add821030b is 50, key is test_row_0/B:col10/1733771157973/Put/seqid=0 2024-12-09T19:05:58,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742282_1458 (size=12001) 2024-12-09T19:05:58,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/925daa12a73a4c458d5aa8add821030b 2024-12-09T19:05:58,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/7d2715d90d704127975497c8af3d649d is 50, key is test_row_0/C:col10/1733771157973/Put/seqid=0 2024-12-09T19:05:58,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742283_1459 (size=12001) 2024-12-09T19:05:58,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:58,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:58,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:58,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:59,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:59,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:59,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:59,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:05:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771219134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771219134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771219134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771219134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:05:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771219135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:05:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:05:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:05:59,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:05:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:59,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:05:59,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/7d2715d90d704127975497c8af3d649d 2024-12-09T19:05:59,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/a93d9abd609e4c8a8294fe75a0667be2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2 2024-12-09T19:05:59,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2, entries=150, sequenceid=16, filesize=30.2 K 2024-12-09T19:05:59,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/925daa12a73a4c458d5aa8add821030b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/925daa12a73a4c458d5aa8add821030b 2024-12-09T19:05:59,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/925daa12a73a4c458d5aa8add821030b, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T19:05:59,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/7d2715d90d704127975497c8af3d649d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/7d2715d90d704127975497c8af3d649d 2024-12-09T19:05:59,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/7d2715d90d704127975497c8af3d649d, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T19:05:59,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for cfa5a71b9d5357947521a419cca3f2d4 in 1308ms, sequenceid=16, compaction requested=false 2024-12-09T19:05:59,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:05:59,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-09T19:05:59,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:05:59,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:05:59,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120927db9b26e0c142efba645bda42fd6f2f_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771157995/Put/seqid=0 2024-12-09T19:05:59,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742284_1460 (size=12154) 2024-12-09T19:05:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:05:59,760 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120927db9b26e0c142efba645bda42fd6f2f_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120927db9b26e0c142efba645bda42fd6f2f_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:05:59,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c509290b24444913a6d12d9be9aa84c5, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:05:59,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c509290b24444913a6d12d9be9aa84c5 is 175, key is test_row_0/A:col10/1733771157995/Put/seqid=0 2024-12-09T19:05:59,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742285_1461 (size=30955) 2024-12-09T19:06:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:06:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:00,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771220154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771220154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,156 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,156 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771220154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771220155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771220156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,167 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c509290b24444913a6d12d9be9aa84c5 2024-12-09T19:06:00,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/5a2dcf7f836045bf8b6ef42327d8bfae is 50, key is test_row_0/B:col10/1733771157995/Put/seqid=0 2024-12-09T19:06:00,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742286_1462 (size=12001) 2024-12-09T19:06:00,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771220264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771220264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771220264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771220475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771220475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771220475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,576 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/5a2dcf7f836045bf8b6ef42327d8bfae 2024-12-09T19:06:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/f18f823aa1e448e5b6420bbf28a852d7 is 50, key is test_row_0/C:col10/1733771157995/Put/seqid=0 2024-12-09T19:06:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742287_1463 (size=12001) 2024-12-09T19:06:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771220784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771220784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771220784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:00,986 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/f18f823aa1e448e5b6420bbf28a852d7 2024-12-09T19:06:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c509290b24444913a6d12d9be9aa84c5 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5 2024-12-09T19:06:00,992 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5, entries=150, sequenceid=41, filesize=30.2 K 2024-12-09T19:06:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/5a2dcf7f836045bf8b6ef42327d8bfae as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5a2dcf7f836045bf8b6ef42327d8bfae 2024-12-09T19:06:00,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5a2dcf7f836045bf8b6ef42327d8bfae, entries=150, sequenceid=41, filesize=11.7 K 2024-12-09T19:06:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/f18f823aa1e448e5b6420bbf28a852d7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/f18f823aa1e448e5b6420bbf28a852d7 2024-12-09T19:06:00,999 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/f18f823aa1e448e5b6420bbf28a852d7, entries=150, sequenceid=41, filesize=11.7 K 2024-12-09T19:06:01,000 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for cfa5a71b9d5357947521a419cca3f2d4 in 1652ms, sequenceid=41, compaction requested=false 2024-12-09T19:06:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:01,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-12-09T19:06:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-12-09T19:06:01,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-09T19:06:01,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0290 sec 2024-12-09T19:06:01,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 3.0340 sec 2024-12-09T19:06:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:01,297 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:01,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:06:01,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c72776b2d24e4b6a8b13790adf5f8f9f_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:01,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742288_1464 (size=17034) 2024-12-09T19:06:01,308 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:01,311 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c72776b2d24e4b6a8b13790adf5f8f9f_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c72776b2d24e4b6a8b13790adf5f8f9f_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:01,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/51b6ccf8a43d45ae8b1c200638743024, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:01,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/51b6ccf8a43d45ae8b1c200638743024 is 175, key is test_row_0/A:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:01,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742289_1465 (size=48139) 2024-12-09T19:06:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771221323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771221326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771221327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771221434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771221434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771221434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771221645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771221645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771221645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,716 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/51b6ccf8a43d45ae8b1c200638743024 2024-12-09T19:06:01,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/ec64aa41a2914f44ba860abde87f3299 is 50, key is test_row_0/B:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:01,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742290_1466 (size=12001) 2024-12-09T19:06:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771221954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771221954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771221954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:02,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/ec64aa41a2914f44ba860abde87f3299 2024-12-09T19:06:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-09T19:06:02,125 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:02,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-12-09T19:06:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-09T19:06:02,127 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:02,127 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:02,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:02,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/fb9970668c8e4bd896d7f0f409e4f7fb is 50, key is test_row_0/C:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742291_1467 (size=12001) 2024-12-09T19:06:02,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771222164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:02,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771222164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:02,166 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:02,166 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-09T19:06:02,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-09T19:06:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:02,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-09T19:06:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:02,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-09T19:06:02,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771222464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:02,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771222464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771222464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:02,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/fb9970668c8e4bd896d7f0f409e4f7fb 2024-12-09T19:06:02,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/51b6ccf8a43d45ae8b1c200638743024 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024 2024-12-09T19:06:02,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024, entries=250, sequenceid=56, filesize=47.0 K 2024-12-09T19:06:02,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/ec64aa41a2914f44ba860abde87f3299 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/ec64aa41a2914f44ba860abde87f3299 2024-12-09T19:06:02,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/ec64aa41a2914f44ba860abde87f3299, entries=150, sequenceid=56, filesize=11.7 K 2024-12-09T19:06:02,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/fb9970668c8e4bd896d7f0f409e4f7fb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/fb9970668c8e4bd896d7f0f409e4f7fb 2024-12-09T19:06:02,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/fb9970668c8e4bd896d7f0f409e4f7fb, entries=150, sequenceid=56, filesize=11.7 K 2024-12-09T19:06:02,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for cfa5a71b9d5357947521a419cca3f2d4 in 1252ms, sequenceid=56, compaction requested=true 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:06:02,549 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:06:02,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:02,549 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:02,550 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:02,550 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:02,550 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/A is initiating minor compaction (all files) 2024-12-09T19:06:02,550 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/B is initiating minor compaction (all files) 2024-12-09T19:06:02,550 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/A in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,550 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/B in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,550 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=107.5 K 2024-12-09T19:06:02,550 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/925daa12a73a4c458d5aa8add821030b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5a2dcf7f836045bf8b6ef42327d8bfae, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/ec64aa41a2914f44ba860abde87f3299] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=35.2 K 2024-12-09T19:06:02,550 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,550 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024] 2024-12-09T19:06:02,551 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 925daa12a73a4c458d5aa8add821030b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733771157973 2024-12-09T19:06:02,551 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93d9abd609e4c8a8294fe75a0667be2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733771157973 2024-12-09T19:06:02,551 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2dcf7f836045bf8b6ef42327d8bfae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733771157993 2024-12-09T19:06:02,551 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c509290b24444913a6d12d9be9aa84c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733771157993 2024-12-09T19:06:02,552 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b6ccf8a43d45ae8b1c200638743024, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733771160154 2024-12-09T19:06:02,552 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ec64aa41a2914f44ba860abde87f3299, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733771160154 2024-12-09T19:06:02,557 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:02,558 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209be054eb817b64663990718ff64989b21_cfa5a71b9d5357947521a419cca3f2d4 store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:02,558 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:02,559 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/67e5dc4dfd9a436e8c25efd1375f595c is 50, key is test_row_0/B:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:02,559 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209be054eb817b64663990718ff64989b21_cfa5a71b9d5357947521a419cca3f2d4, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:02,559 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209be054eb817b64663990718ff64989b21_cfa5a71b9d5357947521a419cca3f2d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:02,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742292_1468 (size=12104) 2024-12-09T19:06:02,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742293_1469 (size=4469) 2024-12-09T19:06:02,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-09T19:06:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,585 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-09T19:06:02,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120989424fd23ed84e4cbc06f42b75b05111_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771161325/Put/seqid=0 2024-12-09T19:06:02,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742294_1470 (size=12154) 2024-12-09T19:06:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-09T19:06:02,972 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/67e5dc4dfd9a436e8c25efd1375f595c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/67e5dc4dfd9a436e8c25efd1375f595c 2024-12-09T19:06:02,973 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#A#compaction#399 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:02,973 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/10772f0f3de04f3cb8914639eee7a2e1 is 175, key is test_row_0/A:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:02,976 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/B of cfa5a71b9d5357947521a419cca3f2d4 into 67e5dc4dfd9a436e8c25efd1375f595c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:02,976 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:02,976 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/B, priority=13, startTime=1733771162549; duration=0sec 2024-12-09T19:06:02,976 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:02,976 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:B 2024-12-09T19:06:02,977 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:02,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742295_1471 (size=31058) 2024-12-09T19:06:02,978 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:02,978 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/C is initiating minor compaction (all files) 2024-12-09T19:06:02,978 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/C in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:02,978 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/7d2715d90d704127975497c8af3d649d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/f18f823aa1e448e5b6420bbf28a852d7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/fb9970668c8e4bd896d7f0f409e4f7fb] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=35.2 K 2024-12-09T19:06:02,978 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2715d90d704127975497c8af3d649d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733771157973 2024-12-09T19:06:02,978 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting f18f823aa1e448e5b6420bbf28a852d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733771157993 2024-12-09T19:06:02,979 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9970668c8e4bd896d7f0f409e4f7fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733771160154 2024-12-09T19:06:02,981 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/10772f0f3de04f3cb8914639eee7a2e1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/10772f0f3de04f3cb8914639eee7a2e1 2024-12-09T19:06:02,985 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:02,985 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/d473f81e358d4f81a7466767ada34678 is 50, key is test_row_0/C:col10/1733771161298/Put/seqid=0 2024-12-09T19:06:02,987 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/A of cfa5a71b9d5357947521a419cca3f2d4 into 10772f0f3de04f3cb8914639eee7a2e1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:02,987 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:02,987 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/A, priority=13, startTime=1733771162549; duration=0sec 2024-12-09T19:06:02,988 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:02,988 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:A 2024-12-09T19:06:02,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742296_1472 (size=12104) 2024-12-09T19:06:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:02,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120989424fd23ed84e4cbc06f42b75b05111_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120989424fd23ed84e4cbc06f42b75b05111_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:02,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/3e15a21c1f294227b74faefeb73555ea, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:02,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/3e15a21c1f294227b74faefeb73555ea is 175, key is test_row_0/A:col10/1733771161325/Put/seqid=0 2024-12-09T19:06:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742297_1473 (size=30955) 2024-12-09T19:06:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-09T19:06:03,393 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T19:06:03,400 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/d473f81e358d4f81a7466767ada34678 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/d473f81e358d4f81a7466767ada34678 2024-12-09T19:06:03,401 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/3e15a21c1f294227b74faefeb73555ea 2024-12-09T19:06:03,407 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/C of cfa5a71b9d5357947521a419cca3f2d4 into d473f81e358d4f81a7466767ada34678(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:03,407 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:03,407 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/C, priority=13, startTime=1733771162549; duration=0sec 2024-12-09T19:06:03,407 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:03,408 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:C 2024-12-09T19:06:03,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/dbc4690200864185b74feaf80f773771 is 50, key is test_row_0/B:col10/1733771161325/Put/seqid=0 2024-12-09T19:06:03,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742298_1474 (size=12001) 2024-12-09T19:06:03,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771223494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771223495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771223495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771223604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771223604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771223604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771223814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771223814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771223814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:03,825 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/dbc4690200864185b74feaf80f773771 2024-12-09T19:06:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/1bca99463da8424daad26c0ba94b2009 is 50, key is test_row_0/C:col10/1733771161325/Put/seqid=0 2024-12-09T19:06:03,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742299_1475 (size=12001) 2024-12-09T19:06:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771224124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771224124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771224124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,243 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/1bca99463da8424daad26c0ba94b2009 2024-12-09T19:06:04,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/3e15a21c1f294227b74faefeb73555ea as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea 2024-12-09T19:06:04,249 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea, entries=150, sequenceid=78, filesize=30.2 K 2024-12-09T19:06:04,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/dbc4690200864185b74feaf80f773771 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/dbc4690200864185b74feaf80f773771 2024-12-09T19:06:04,252 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/dbc4690200864185b74feaf80f773771, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:06:04,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/1bca99463da8424daad26c0ba94b2009 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/1bca99463da8424daad26c0ba94b2009 2024-12-09T19:06:04,256 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/1bca99463da8424daad26c0ba94b2009, entries=150, sequenceid=78, filesize=11.7 K 2024-12-09T19:06:04,256 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for cfa5a71b9d5357947521a419cca3f2d4 in 1671ms, sequenceid=78, compaction requested=false 2024-12-09T19:06:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:04,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-12-09T19:06:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-12-09T19:06:04,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-09T19:06:04,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1300 sec 2024-12-09T19:06:04,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.1350 sec 2024-12-09T19:06:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-09T19:06:04,275 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:04,275 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-09T19:06:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T19:06:04,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:04,278 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:04,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T19:06:04,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-09T19:06:04,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:04,430 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-09T19:06:04,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120984446224e47f4e9d9f18e9a8727657d3_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771163493/Put/seqid=0 2024-12-09T19:06:04,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742300_1476 (size=12154) 2024-12-09T19:06:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T19:06:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:04,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771224655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771224657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771224658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771224765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771224765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771224765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:04,841 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120984446224e47f4e9d9f18e9a8727657d3_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120984446224e47f4e9d9f18e9a8727657d3_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/59742770696a4cc3bc06ebe0d15beb13, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/59742770696a4cc3bc06ebe0d15beb13 is 175, key is test_row_0/A:col10/1733771163493/Put/seqid=0 2024-12-09T19:06:04,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742301_1477 (size=30955) 2024-12-09T19:06:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T19:06:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771224975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771224975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771224975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:05,245 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/59742770696a4cc3bc06ebe0d15beb13 2024-12-09T19:06:05,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/2251e6b003f840a3a865ffe651021c35 is 50, key is test_row_0/B:col10/1733771163493/Put/seqid=0 2024-12-09T19:06:05,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742302_1478 (size=12001) 2024-12-09T19:06:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771225284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771225284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771225284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T19:06:05,654 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/2251e6b003f840a3a865ffe651021c35 2024-12-09T19:06:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/538adfcc9a3844088b4b69959be27a00 is 50, key is test_row_0/C:col10/1733771163493/Put/seqid=0 2024-12-09T19:06:05,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742303_1479 (size=12001) 2024-12-09T19:06:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771225794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771225794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771225794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,063 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/538adfcc9a3844088b4b69959be27a00 2024-12-09T19:06:06,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/59742770696a4cc3bc06ebe0d15beb13 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13 2024-12-09T19:06:06,070 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13, entries=150, sequenceid=96, filesize=30.2 K 2024-12-09T19:06:06,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/2251e6b003f840a3a865ffe651021c35 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/2251e6b003f840a3a865ffe651021c35 2024-12-09T19:06:06,074 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/2251e6b003f840a3a865ffe651021c35, entries=150, sequenceid=96, filesize=11.7 K 2024-12-09T19:06:06,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/538adfcc9a3844088b4b69959be27a00 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/538adfcc9a3844088b4b69959be27a00 2024-12-09T19:06:06,077 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/538adfcc9a3844088b4b69959be27a00, entries=150, sequenceid=96, filesize=11.7 K 2024-12-09T19:06:06,078 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for cfa5a71b9d5357947521a419cca3f2d4 in 1648ms, sequenceid=96, compaction requested=true 2024-12-09T19:06:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-09T19:06:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-09T19:06:06,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-09T19:06:06,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-09T19:06:06,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.8060 sec 2024-12-09T19:06:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:06,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:06,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-09T19:06:06,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a0527c6f41ed48a88c5d2d8f8ebd6125_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:06,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742304_1480 (size=12154) 2024-12-09T19:06:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771226198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771226200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771226304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771226304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-09T19:06:06,425 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:06,425 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-09T19:06:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T19:06:06,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:06,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:06,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771226514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771226514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T19:06:06,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,585 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:06,588 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209a0527c6f41ed48a88c5d2d8f8ebd6125_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a0527c6f41ed48a88c5d2d8f8ebd6125_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:06,588 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c7b76b5dadd04f5ea12439f5e3957c20, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:06,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c7b76b5dadd04f5ea12439f5e3957c20 is 175, key is test_row_0/A:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:06,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742305_1481 (size=30955) 2024-12-09T19:06:06,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c7b76b5dadd04f5ea12439f5e3957c20 2024-12-09T19:06:06,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/bad3e5cdd816452395b98d40137683c8 is 50, key is test_row_0/B:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:06,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742306_1482 (size=12001) 2024-12-09T19:06:06,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T19:06:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771226804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771226804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,807 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,807 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771226814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,817 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771226824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771226824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:06,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:06,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/bad3e5cdd816452395b98d40137683c8 2024-12-09T19:06:07,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/63090fa972f4407aa1a48e5808eba3ae is 50, key is test_row_0/C:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742307_1483 (size=12001) 2024-12-09T19:06:07,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T19:06:07,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771227334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:07,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771227334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:07,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:07,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:07,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/63090fa972f4407aa1a48e5808eba3ae 2024-12-09T19:06:07,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/c7b76b5dadd04f5ea12439f5e3957c20 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20 2024-12-09T19:06:07,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20, entries=150, sequenceid=118, filesize=30.2 K 2024-12-09T19:06:07,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/bad3e5cdd816452395b98d40137683c8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/bad3e5cdd816452395b98d40137683c8 2024-12-09T19:06:07,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/bad3e5cdd816452395b98d40137683c8, entries=150, sequenceid=118, filesize=11.7 K 2024-12-09T19:06:07,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/63090fa972f4407aa1a48e5808eba3ae as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/63090fa972f4407aa1a48e5808eba3ae 2024-12-09T19:06:07,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/63090fa972f4407aa1a48e5808eba3ae, entries=150, sequenceid=118, filesize=11.7 K 2024-12-09T19:06:07,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for cfa5a71b9d5357947521a419cca3f2d4 in 1250ms, sequenceid=118, compaction requested=true 2024-12-09T19:06:07,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:07,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:06:07,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:07,426 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:06:07,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:06:07,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:07,427 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:06:07,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:06:07,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:07,427 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:06:07,427 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:06:07,427 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/A is initiating minor compaction (all files) 2024-12-09T19:06:07,427 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/B is initiating minor compaction (all files) 2024-12-09T19:06:07,427 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/A in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,427 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/B in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,427 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/10772f0f3de04f3cb8914639eee7a2e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=121.0 K 2024-12-09T19:06:07,427 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/67e5dc4dfd9a436e8c25efd1375f595c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/dbc4690200864185b74feaf80f773771, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/2251e6b003f840a3a865ffe651021c35, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/bad3e5cdd816452395b98d40137683c8] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=47.0 K 2024-12-09T19:06:07,427 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,427 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/10772f0f3de04f3cb8914639eee7a2e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20] 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e5dc4dfd9a436e8c25efd1375f595c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733771160154 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10772f0f3de04f3cb8914639eee7a2e1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733771160154 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting dbc4690200864185b74feaf80f773771, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771161317 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e15a21c1f294227b74faefeb73555ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771161317 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2251e6b003f840a3a865ffe651021c35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733771163486 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59742770696a4cc3bc06ebe0d15beb13, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733771163486 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting bad3e5cdd816452395b98d40137683c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733771164655 2024-12-09T19:06:07,428 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b76b5dadd04f5ea12439f5e3957c20, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733771164655 2024-12-09T19:06:07,434 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:07,434 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/5b265ab34ae34cada6983e4cc5499ccb is 50, key is test_row_0/B:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:07,435 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:07,437 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209ed8b4342f99f4da0adbcbb2d8350328e_cfa5a71b9d5357947521a419cca3f2d4 store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:07,438 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209ed8b4342f99f4da0adbcbb2d8350328e_cfa5a71b9d5357947521a419cca3f2d4, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:07,438 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ed8b4342f99f4da0adbcbb2d8350328e_cfa5a71b9d5357947521a419cca3f2d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742309_1485 (size=4469) 2024-12-09T19:06:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742308_1484 (size=12241) 2024-12-09T19:06:07,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-09T19:06:07,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,494 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-09T19:06:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094eb9a987e72b4a5e8097a5c9dcdaaddc_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771166197/Put/seqid=0 2024-12-09T19:06:07,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742310_1486 (size=12254) 2024-12-09T19:06:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T19:06:07,885 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#A#compaction#412 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:07,885 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/6fb8cae62a4f42c89dd3e820d75fcc28 is 175, key is test_row_0/A:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:07,888 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/5b265ab34ae34cada6983e4cc5499ccb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5b265ab34ae34cada6983e4cc5499ccb 2024-12-09T19:06:07,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742311_1487 (size=31195) 2024-12-09T19:06:07,892 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/B of cfa5a71b9d5357947521a419cca3f2d4 into 5b265ab34ae34cada6983e4cc5499ccb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:07,892 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:07,892 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/B, priority=12, startTime=1733771167426; duration=0sec 2024-12-09T19:06:07,892 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:07,892 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:B 2024-12-09T19:06:07,892 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T19:06:07,893 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T19:06:07,893 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/C is initiating minor compaction (all files) 2024-12-09T19:06:07,893 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/C in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:07,893 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/d473f81e358d4f81a7466767ada34678, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/1bca99463da8424daad26c0ba94b2009, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/538adfcc9a3844088b4b69959be27a00, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/63090fa972f4407aa1a48e5808eba3ae] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=47.0 K 2024-12-09T19:06:07,894 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting d473f81e358d4f81a7466767ada34678, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733771160154 2024-12-09T19:06:07,894 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bca99463da8424daad26c0ba94b2009, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733771161317 2024-12-09T19:06:07,894 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 538adfcc9a3844088b4b69959be27a00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733771163486 2024-12-09T19:06:07,895 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 63090fa972f4407aa1a48e5808eba3ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733771164655 2024-12-09T19:06:07,900 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:07,900 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/6279f1aca484464d9980496f4fa28581 is 50, key is test_row_0/C:col10/1733771166175/Put/seqid=0 2024-12-09T19:06:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:07,905 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094eb9a987e72b4a5e8097a5c9dcdaaddc_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094eb9a987e72b4a5e8097a5c9dcdaaddc_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/63c12d85e03e4e179d9cab81bc6d78f8, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/63c12d85e03e4e179d9cab81bc6d78f8 is 175, key is test_row_0/A:col10/1733771166197/Put/seqid=0 2024-12-09T19:06:07,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742312_1488 (size=12241) 2024-12-09T19:06:07,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742313_1489 (size=31055) 2024-12-09T19:06:07,911 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/63c12d85e03e4e179d9cab81bc6d78f8 2024-12-09T19:06:07,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/700fdbb2770a48939e149f99d01884ed is 50, key is test_row_0/B:col10/1733771166197/Put/seqid=0 2024-12-09T19:06:07,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742314_1490 (size=12101) 2024-12-09T19:06:08,293 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/6fb8cae62a4f42c89dd3e820d75fcc28 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/6fb8cae62a4f42c89dd3e820d75fcc28 2024-12-09T19:06:08,296 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/A of cfa5a71b9d5357947521a419cca3f2d4 into 6fb8cae62a4f42c89dd3e820d75fcc28(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:08,296 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:08,296 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/A, priority=12, startTime=1733771167426; duration=0sec 2024-12-09T19:06:08,296 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:08,296 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:A 2024-12-09T19:06:08,314 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/6279f1aca484464d9980496f4fa28581 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/6279f1aca484464d9980496f4fa28581 2024-12-09T19:06:08,317 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/C of cfa5a71b9d5357947521a419cca3f2d4 into 6279f1aca484464d9980496f4fa28581(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:08,317 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:08,317 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/C, priority=12, startTime=1733771167427; duration=0sec 2024-12-09T19:06:08,317 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:08,317 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:C 2024-12-09T19:06:08,321 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/700fdbb2770a48939e149f99d01884ed 2024-12-09T19:06:08,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/ee6ed7a3ef5a4dc8a705a0ec51a80394 is 50, key is test_row_0/C:col10/1733771166197/Put/seqid=0 2024-12-09T19:06:08,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742315_1491 (size=12101) 2024-12-09T19:06:08,330 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/ee6ed7a3ef5a4dc8a705a0ec51a80394 2024-12-09T19:06:08,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/63c12d85e03e4e179d9cab81bc6d78f8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8 2024-12-09T19:06:08,335 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8, entries=150, sequenceid=132, filesize=30.3 K 2024-12-09T19:06:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/700fdbb2770a48939e149f99d01884ed as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/700fdbb2770a48939e149f99d01884ed 2024-12-09T19:06:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,339 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/700fdbb2770a48939e149f99d01884ed, entries=150, sequenceid=132, filesize=11.8 K 2024-12-09T19:06:08,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/ee6ed7a3ef5a4dc8a705a0ec51a80394 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/ee6ed7a3ef5a4dc8a705a0ec51a80394 2024-12-09T19:06:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/ee6ed7a3ef5a4dc8a705a0ec51a80394, entries=150, sequenceid=132, filesize=11.8 K 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for cfa5a71b9d5357947521a419cca3f2d4 in 849ms, sequenceid=132, compaction requested=false 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-09T19:06:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-09T19:06:08,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9170 sec 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.9210 sec 2024-12-09T19:06:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:08,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:08,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:06:08,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099d1aa228e6fd49228a8454d9d95bfb08_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:08,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742316_1492 (size=12304) 2024-12-09T19:06:08,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771228453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771228457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771228564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771228565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T19:06:08,574 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:08,574 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-09T19:06:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T19:06:08,576 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:08,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:08,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T19:06:08,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:08,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:08,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:08,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:08,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:08,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:08,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771228775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771228775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771228824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771228824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771228824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:08,827 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:08,827 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,828 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:08,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:08,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099d1aa228e6fd49228a8454d9d95bfb08_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d1aa228e6fd49228a8454d9d95bfb08_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:08,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/d9100d47edb24a3c8864dce30516750b, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:08,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/d9100d47edb24a3c8864dce30516750b is 175, key is test_row_0/A:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:08,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742317_1493 (size=31105) 2024-12-09T19:06:08,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:08,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:08,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T19:06:09,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771229084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771229084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:09,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T19:06:09,236 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/d9100d47edb24a3c8864dce30516750b 2024-12-09T19:06:09,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/138202461b614fdd84b295e7e401b055 is 50, key is test_row_0/B:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:09,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742318_1494 (size=12151) 2024-12-09T19:06:09,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771229594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771229594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:09,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/138202461b614fdd84b295e7e401b055 2024-12-09T19:06:09,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/0d7be3ede7354fc1baf56bbc60bd10fe is 50, key is test_row_0/C:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:09,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742319_1495 (size=12151) 2024-12-09T19:06:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T19:06:09,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:09,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:09,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:10,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/0d7be3ede7354fc1baf56bbc60bd10fe 2024-12-09T19:06:10,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/d9100d47edb24a3c8864dce30516750b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b 2024-12-09T19:06:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b, entries=150, sequenceid=146, filesize=30.4 K 2024-12-09T19:06:10,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/138202461b614fdd84b295e7e401b055 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/138202461b614fdd84b295e7e401b055 2024-12-09T19:06:10,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/138202461b614fdd84b295e7e401b055, entries=150, sequenceid=146, filesize=11.9 K 2024-12-09T19:06:10,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/0d7be3ede7354fc1baf56bbc60bd10fe as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/0d7be3ede7354fc1baf56bbc60bd10fe 2024-12-09T19:06:10,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/0d7be3ede7354fc1baf56bbc60bd10fe, entries=150, sequenceid=146, filesize=11.9 K 2024-12-09T19:06:10,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cfa5a71b9d5357947521a419cca3f2d4 in 1658ms, sequenceid=146, compaction requested=true 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:10,069 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:10,069 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:06:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/A is initiating minor compaction (all files) 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/B is initiating minor compaction (all files) 2024-12-09T19:06:10,070 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/B in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,070 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/A in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,070 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/6fb8cae62a4f42c89dd3e820d75fcc28, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=91.2 K 2024-12-09T19:06:10,070 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5b265ab34ae34cada6983e4cc5499ccb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/700fdbb2770a48939e149f99d01884ed, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/138202461b614fdd84b295e7e401b055] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=35.6 K 2024-12-09T19:06:10,070 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/6fb8cae62a4f42c89dd3e820d75fcc28, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b] 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b265ab34ae34cada6983e4cc5499ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733771164655 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fb8cae62a4f42c89dd3e820d75fcc28, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733771164655 2024-12-09T19:06:10,070 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 700fdbb2770a48939e149f99d01884ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771166193 2024-12-09T19:06:10,071 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c12d85e03e4e179d9cab81bc6d78f8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771166193 2024-12-09T19:06:10,071 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 138202461b614fdd84b295e7e401b055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733771168403 2024-12-09T19:06:10,071 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9100d47edb24a3c8864dce30516750b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733771168403 2024-12-09T19:06:10,084 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:10,088 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:10,089 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/3a0d3fe34d224a859a5ee18a8a7852c8 is 50, key is test_row_0/B:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:10,100 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412091c9fe33dd9cd42e9b0941b67f6acefb0_cfa5a71b9d5357947521a419cca3f2d4 store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:10,101 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412091c9fe33dd9cd42e9b0941b67f6acefb0_cfa5a71b9d5357947521a419cca3f2d4, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:10,102 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091c9fe33dd9cd42e9b0941b67f6acefb0_cfa5a71b9d5357947521a419cca3f2d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:10,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742320_1496 (size=12493) 2024-12-09T19:06:10,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-09T19:06:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,103 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:06:10,109 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/3a0d3fe34d224a859a5ee18a8a7852c8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/3a0d3fe34d224a859a5ee18a8a7852c8 2024-12-09T19:06:10,114 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/B of cfa5a71b9d5357947521a419cca3f2d4 into 3a0d3fe34d224a859a5ee18a8a7852c8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:10,114 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:10,114 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/B, priority=13, startTime=1733771170069; duration=0sec 2024-12-09T19:06:10,114 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:10,114 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:B 2024-12-09T19:06:10,114 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:10,115 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:10,115 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/C is initiating minor compaction (all files) 2024-12-09T19:06:10,115 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/C in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,115 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/6279f1aca484464d9980496f4fa28581, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/ee6ed7a3ef5a4dc8a705a0ec51a80394, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/0d7be3ede7354fc1baf56bbc60bd10fe] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=35.6 K 2024-12-09T19:06:10,115 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 6279f1aca484464d9980496f4fa28581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733771164655 2024-12-09T19:06:10,115 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6ed7a3ef5a4dc8a705a0ec51a80394, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733771166193 2024-12-09T19:06:10,116 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d7be3ede7354fc1baf56bbc60bd10fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733771168403 2024-12-09T19:06:10,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e3f934c6c8714f87b2ce4a34ca1894e3_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771168436/Put/seqid=0 2024-12-09T19:06:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742321_1497 (size=4469) 2024-12-09T19:06:10,123 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:10,123 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/b7740a295bdd4af5b98fa44482dbc993 is 50, key is test_row_0/C:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:10,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742322_1498 (size=12304) 2024-12-09T19:06:10,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,129 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e3f934c6c8714f87b2ce4a34ca1894e3_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e3f934c6c8714f87b2ce4a34ca1894e3_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/f43ac91c68ba40779bb065067bbd25b4, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:10,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/f43ac91c68ba40779bb065067bbd25b4 is 175, key is test_row_0/A:col10/1733771168436/Put/seqid=0 2024-12-09T19:06:10,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742323_1499 (size=12493) 2024-12-09T19:06:10,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742324_1500 (size=31105) 2024-12-09T19:06:10,145 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/f43ac91c68ba40779bb065067bbd25b4 2024-12-09T19:06:10,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/9335d39659f740c583c4a26aec14abb4 is 50, key is test_row_0/B:col10/1733771168436/Put/seqid=0 2024-12-09T19:06:10,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742325_1501 (size=12151) 2024-12-09T19:06:10,522 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#A#compaction#420 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:10,522 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/136149b126b941898b67045352f7be88 is 175, key is test_row_0/A:col10/1733771168408/Put/seqid=0 2024-12-09T19:06:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742326_1502 (size=31447) 2024-12-09T19:06:10,544 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/b7740a295bdd4af5b98fa44482dbc993 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b7740a295bdd4af5b98fa44482dbc993 2024-12-09T19:06:10,548 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/C of cfa5a71b9d5357947521a419cca3f2d4 into b7740a295bdd4af5b98fa44482dbc993(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:10,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:10,548 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/C, priority=13, startTime=1733771170069; duration=0sec 2024-12-09T19:06:10,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:10,548 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:C 2024-12-09T19:06:10,557 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/9335d39659f740c583c4a26aec14abb4 2024-12-09T19:06:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/50ebc493d1a64395adf713ed0534f8c7 is 50, key is test_row_0/C:col10/1733771168436/Put/seqid=0 2024-12-09T19:06:10,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742327_1503 (size=12151) 2024-12-09T19:06:10,566 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/50ebc493d1a64395adf713ed0534f8c7 2024-12-09T19:06:10,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/f43ac91c68ba40779bb065067bbd25b4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4 2024-12-09T19:06:10,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4, entries=150, sequenceid=171, filesize=30.4 K 2024-12-09T19:06:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/9335d39659f740c583c4a26aec14abb4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/9335d39659f740c583c4a26aec14abb4 2024-12-09T19:06:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/9335d39659f740c583c4a26aec14abb4, entries=150, sequenceid=171, filesize=11.9 K 2024-12-09T19:06:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/50ebc493d1a64395adf713ed0534f8c7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/50ebc493d1a64395adf713ed0534f8c7 2024-12-09T19:06:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,585 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/50ebc493d1a64395adf713ed0534f8c7, entries=150, sequenceid=171, filesize=11.9 K 2024-12-09T19:06:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for cfa5a71b9d5357947521a419cca3f2d4 in 483ms, sequenceid=171, compaction requested=false 2024-12-09T19:06:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-09T19:06:10,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.0160 sec 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:10,658 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:10,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:06:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:10,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e35b33ec3439446090c355720db69a5f_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:10,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742328_1504 (size=24758) 2024-12-09T19:06:10,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771230708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:10,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:10,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771230709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-09T19:06:10,724 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:10,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-09T19:06:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T19:06:10,726 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:10,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:10,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:10,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771230815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:10,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771230815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T19:06:10,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:10,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:10,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:10,931 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/136149b126b941898b67045352f7be88 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/136149b126b941898b67045352f7be88 2024-12-09T19:06:10,934 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/A of cfa5a71b9d5357947521a419cca3f2d4 into 136149b126b941898b67045352f7be88(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:10,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:10,934 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/A, priority=13, startTime=1733771170069; duration=0sec 2024-12-09T19:06:10,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:10,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:A 2024-12-09T19:06:11,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771231024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:11,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771231024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:11,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T19:06:11,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:11,088 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209e35b33ec3439446090c355720db69a5f_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e35b33ec3439446090c355720db69a5f_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:11,089 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/077ef55a6bac40c1a2b1c101bd04aea6, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:11,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/077ef55a6bac40c1a2b1c101bd04aea6 is 175, key is test_row_0/A:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742329_1505 (size=74395) 2024-12-09T19:06:11,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/077ef55a6bac40c1a2b1c101bd04aea6 2024-12-09T19:06:11,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/94a058d810d1427e8b32cfcf35e14808 is 50, key is test_row_0/B:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742330_1506 (size=12151) 2024-12-09T19:06:11,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:11,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:11,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771231334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:11,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771231334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T19:06:11,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/94a058d810d1427e8b32cfcf35e14808 2024-12-09T19:06:11,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/b2f000e300f24b0bb56897315ced6013 is 50, key is test_row_0/C:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:11,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742331_1507 (size=12151) 2024-12-09T19:06:11,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:11,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771231844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:11,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771231844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:11,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T19:06:11,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/b2f000e300f24b0bb56897315ced6013 2024-12-09T19:06:11,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/077ef55a6bac40c1a2b1c101bd04aea6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6 2024-12-09T19:06:11,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6, entries=400, sequenceid=185, filesize=72.7 K 2024-12-09T19:06:11,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/94a058d810d1427e8b32cfcf35e14808 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/94a058d810d1427e8b32cfcf35e14808 2024-12-09T19:06:11,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/94a058d810d1427e8b32cfcf35e14808, entries=150, sequenceid=185, filesize=11.9 K 2024-12-09T19:06:11,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/b2f000e300f24b0bb56897315ced6013 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b2f000e300f24b0bb56897315ced6013 2024-12-09T19:06:11,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b2f000e300f24b0bb56897315ced6013, entries=150, sequenceid=185, filesize=11.9 K 2024-12-09T19:06:11,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cfa5a71b9d5357947521a419cca3f2d4 in 1274ms, sequenceid=185, compaction requested=true 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:11,933 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:06:11,933 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:11,933 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:11,933 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:11,933 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/A is initiating minor compaction (all files) 2024-12-09T19:06:11,933 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/B is initiating minor compaction (all files) 2024-12-09T19:06:11,933 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/B in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,933 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/A in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,934 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/136149b126b941898b67045352f7be88, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=133.7 K 2024-12-09T19:06:11,934 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/3a0d3fe34d224a859a5ee18a8a7852c8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/9335d39659f740c583c4a26aec14abb4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/94a058d810d1427e8b32cfcf35e14808] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=35.9 K 2024-12-09T19:06:11,934 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/136149b126b941898b67045352f7be88, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6] 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0d3fe34d224a859a5ee18a8a7852c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733771168403 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 136149b126b941898b67045352f7be88, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733771168403 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 9335d39659f740c583c4a26aec14abb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733771168436 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting f43ac91c68ba40779bb065067bbd25b4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733771168436 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 94a058d810d1427e8b32cfcf35e14808, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733771170629 2024-12-09T19:06:11,934 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077ef55a6bac40c1a2b1c101bd04aea6, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733771170629 2024-12-09T19:06:11,939 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:11,940 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120953fd891633fc4ec4b2f7ac4068f92902_cfa5a71b9d5357947521a419cca3f2d4 store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:11,941 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:11,941 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/b48aa48b5cef4edaa18e86a54c2e77a1 is 50, key is test_row_0/B:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:11,942 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120953fd891633fc4ec4b2f7ac4068f92902_cfa5a71b9d5357947521a419cca3f2d4, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:11,942 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120953fd891633fc4ec4b2f7ac4068f92902_cfa5a71b9d5357947521a419cca3f2d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:11,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-09T19:06:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:11,944 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-09T19:06:11,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742333_1509 (size=4469) 2024-12-09T19:06:11,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742332_1508 (size=12595) 2024-12-09T19:06:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d91f7f2e21cd4fe78848a2fddf4d8b86_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771170707/Put/seqid=0 2024-12-09T19:06:11,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742334_1510 (size=12304) 2024-12-09T19:06:12,358 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#A#compaction#429 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:12,358 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/e208b22f19b14c69bab461c8ced6e2d2 is 175, key is test_row_0/A:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:12,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742335_1511 (size=31549) 2024-12-09T19:06:12,374 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/b48aa48b5cef4edaa18e86a54c2e77a1 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/b48aa48b5cef4edaa18e86a54c2e77a1 2024-12-09T19:06:12,377 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/B of cfa5a71b9d5357947521a419cca3f2d4 into b48aa48b5cef4edaa18e86a54c2e77a1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:12,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:12,378 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/B, priority=13, startTime=1733771171933; duration=0sec 2024-12-09T19:06:12,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:12,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:B 2024-12-09T19:06:12,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:12,378 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:12,379 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/C is initiating minor compaction (all files) 2024-12-09T19:06:12,379 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/C in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:12,379 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b7740a295bdd4af5b98fa44482dbc993, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/50ebc493d1a64395adf713ed0534f8c7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b2f000e300f24b0bb56897315ced6013] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=35.9 K 2024-12-09T19:06:12,379 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b7740a295bdd4af5b98fa44482dbc993, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733771168403 2024-12-09T19:06:12,379 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ebc493d1a64395adf713ed0534f8c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733771168436 2024-12-09T19:06:12,379 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f000e300f24b0bb56897315ced6013, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733771170629 2024-12-09T19:06:12,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,383 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d91f7f2e21cd4fe78848a2fddf4d8b86_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d91f7f2e21cd4fe78848a2fddf4d8b86_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:12,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/66223ec6c6b54cdfb2c2ad69663d757d, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:12,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/66223ec6c6b54cdfb2c2ad69663d757d is 175, key is test_row_0/A:col10/1733771170707/Put/seqid=0 2024-12-09T19:06:12,385 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:12,386 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/2fd78c19999b4ab591103e3f721484a8 is 50, key is test_row_0/C:col10/1733771170658/Put/seqid=0 2024-12-09T19:06:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742336_1512 (size=31105) 2024-12-09T19:06:12,389 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/66223ec6c6b54cdfb2c2ad69663d757d 2024-12-09T19:06:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742337_1513 (size=12595) 2024-12-09T19:06:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/7b2a219549584a5e82ff0546960e1edb is 50, key is test_row_0/B:col10/1733771170707/Put/seqid=0 2024-12-09T19:06:12,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742338_1514 (size=12151) 2024-12-09T19:06:12,410 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/7b2a219549584a5e82ff0546960e1edb 2024-12-09T19:06:12,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/36207faf2db5440ab766b5ee5d96b3cc is 50, key is test_row_0/C:col10/1733771170707/Put/seqid=0 2024-12-09T19:06:12,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742339_1515 (size=12151) 2024-12-09T19:06:12,766 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/e208b22f19b14c69bab461c8ced6e2d2 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e208b22f19b14c69bab461c8ced6e2d2 2024-12-09T19:06:12,770 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/A of cfa5a71b9d5357947521a419cca3f2d4 into e208b22f19b14c69bab461c8ced6e2d2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:12,770 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:12,770 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/A, priority=13, startTime=1733771171933; duration=0sec 2024-12-09T19:06:12,770 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:12,770 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:A 2024-12-09T19:06:12,809 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/2fd78c19999b4ab591103e3f721484a8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/2fd78c19999b4ab591103e3f721484a8 2024-12-09T19:06:12,812 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/C of cfa5a71b9d5357947521a419cca3f2d4 into 2fd78c19999b4ab591103e3f721484a8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:12,812 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:12,812 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/C, priority=13, startTime=1733771171933; duration=0sec 2024-12-09T19:06:12,812 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:12,812 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:C 2024-12-09T19:06:12,823 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/36207faf2db5440ab766b5ee5d96b3cc 2024-12-09T19:06:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/66223ec6c6b54cdfb2c2ad69663d757d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d 2024-12-09T19:06:12,829 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d, entries=150, sequenceid=210, filesize=30.4 K 2024-12-09T19:06:12,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/7b2a219549584a5e82ff0546960e1edb as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/7b2a219549584a5e82ff0546960e1edb 2024-12-09T19:06:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,833 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/7b2a219549584a5e82ff0546960e1edb, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T19:06:12,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/36207faf2db5440ab766b5ee5d96b3cc as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/36207faf2db5440ab766b5ee5d96b3cc 2024-12-09T19:06:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/36207faf2db5440ab766b5ee5d96b3cc, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for cfa5a71b9d5357947521a419cca3f2d4 in 894ms, sequenceid=210, compaction requested=false 2024-12-09T19:06:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-09T19:06:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-09T19:06:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1120 sec 2024-12-09T19:06:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.1160 sec 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:12,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:12,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-09T19:06:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209445954af124e4007a2253c22d15cd50c_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-09T19:06:12,885 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:12,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:12,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742341_1517 (size=24758) 2024-12-09T19:06:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-09T19:06:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:12,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:12,893 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:12,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771232900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771232901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771232902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:12,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771232903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771232905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:13,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771233015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771233015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771233015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771233015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771233015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771233224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771233224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771233224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771233224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771233224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,288 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:13,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209445954af124e4007a2253c22d15cd50c_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209445954af124e4007a2253c22d15cd50c_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:13,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/b0fbc91c6ee84d778e8fb367d6e39e57, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:13,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/b0fbc91c6ee84d778e8fb367d6e39e57 is 175, key is test_row_0/A:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:13,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742340_1516 (size=74395) 2024-12-09T19:06:13,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:13,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771233535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771233535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771233535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771233535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771233535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:13,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/b0fbc91c6ee84d778e8fb367d6e39e57 2024-12-09T19:06:13,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/0c5f4c16e6ab44ccb79f12defcee06e4 is 50, key is test_row_0/B:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:13,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742342_1518 (size=12151) 2024-12-09T19:06:13,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:13,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:14,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771234045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771234045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:14,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771234045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:14,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771234045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:14,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771234045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:14,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/0c5f4c16e6ab44ccb79f12defcee06e4 2024-12-09T19:06:14,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/696259db58514fa9b861f8e84850f546 is 50, key is test_row_0/C:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:14,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742343_1519 (size=12151) 2024-12-09T19:06:14,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:14,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:14,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/696259db58514fa9b861f8e84850f546 2024-12-09T19:06:14,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/b0fbc91c6ee84d778e8fb367d6e39e57 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57 2024-12-09T19:06:14,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57, entries=400, sequenceid=224, filesize=72.7 K 2024-12-09T19:06:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/0c5f4c16e6ab44ccb79f12defcee06e4 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c5f4c16e6ab44ccb79f12defcee06e4 2024-12-09T19:06:14,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c5f4c16e6ab44ccb79f12defcee06e4, entries=150, sequenceid=224, filesize=11.9 K 2024-12-09T19:06:14,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/696259db58514fa9b861f8e84850f546 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/696259db58514fa9b861f8e84850f546 2024-12-09T19:06:14,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/696259db58514fa9b861f8e84850f546, entries=150, sequenceid=224, filesize=11.9 K 2024-12-09T19:06:14,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cfa5a71b9d5357947521a419cca3f2d4 in 1656ms, sequenceid=224, compaction requested=true 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:14,525 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:06:14,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:14,525 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:14,525 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:14,525 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/A is initiating minor compaction (all files) 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/B is initiating minor compaction (all files) 2024-12-09T19:06:14,526 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/A in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,526 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/B in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,526 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e208b22f19b14c69bab461c8ced6e2d2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=133.8 K 2024-12-09T19:06:14,526 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/b48aa48b5cef4edaa18e86a54c2e77a1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/7b2a219549584a5e82ff0546960e1edb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c5f4c16e6ab44ccb79f12defcee06e4] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=36.0 K 2024-12-09T19:06:14,526 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e208b22f19b14c69bab461c8ced6e2d2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57] 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting b48aa48b5cef4edaa18e86a54c2e77a1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733771170629 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting e208b22f19b14c69bab461c8ced6e2d2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733771170629 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2a219549584a5e82ff0546960e1edb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771170703 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66223ec6c6b54cdfb2c2ad69663d757d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771170703 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5f4c16e6ab44ccb79f12defcee06e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733771172859 2024-12-09T19:06:14,526 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0fbc91c6ee84d778e8fb367d6e39e57, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733771172856 2024-12-09T19:06:14,530 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:14,532 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:14,532 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/0c145e658e1947cfa22cb15a8040e80b is 50, key is test_row_0/B:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:14,532 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209c9213b36310a4e388c4757602b5d9f34_cfa5a71b9d5357947521a419cca3f2d4 store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:14,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209c9213b36310a4e388c4757602b5d9f34_cfa5a71b9d5357947521a419cca3f2d4, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:14,535 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c9213b36310a4e388c4757602b5d9f34_cfa5a71b9d5357947521a419cca3f2d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:14,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742344_1520 (size=12697) 2024-12-09T19:06:14,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742345_1521 (size=4469) 2024-12-09T19:06:14,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-09T19:06:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,569 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-09T19:06:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d9fe7386837454f86d4dfb975aa560a_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771172900/Put/seqid=0 2024-12-09T19:06:14,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742346_1522 (size=12304) 2024-12-09T19:06:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:14,581 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095d9fe7386837454f86d4dfb975aa560a_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d9fe7386837454f86d4dfb975aa560a_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/01ea95014a7a4eafaac5636b68356929, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/01ea95014a7a4eafaac5636b68356929 is 175, key is test_row_0/A:col10/1733771172900/Put/seqid=0 2024-12-09T19:06:14,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742347_1523 (size=31105) 2024-12-09T19:06:14,940 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#A#compaction#439 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:14,940 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/1f49d227ac3d457c832295871b97a7a6 is 175, key is test_row_0/A:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:14,942 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/0c145e658e1947cfa22cb15a8040e80b as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c145e658e1947cfa22cb15a8040e80b 2024-12-09T19:06:14,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742348_1524 (size=31651) 2024-12-09T19:06:14,947 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/B of cfa5a71b9d5357947521a419cca3f2d4 into 0c145e658e1947cfa22cb15a8040e80b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:14,947 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:14,947 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/B, priority=13, startTime=1733771174525; duration=0sec 2024-12-09T19:06:14,947 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:14,947 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:B 2024-12-09T19:06:14,947 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:14,948 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/1f49d227ac3d457c832295871b97a7a6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/1f49d227ac3d457c832295871b97a7a6 2024-12-09T19:06:14,948 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:14,948 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/C is initiating minor compaction (all files) 2024-12-09T19:06:14,948 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/C in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:14,948 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/2fd78c19999b4ab591103e3f721484a8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/36207faf2db5440ab766b5ee5d96b3cc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/696259db58514fa9b861f8e84850f546] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=36.0 K 2024-12-09T19:06:14,949 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd78c19999b4ab591103e3f721484a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733771170629 2024-12-09T19:06:14,949 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 36207faf2db5440ab766b5ee5d96b3cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733771170703 2024-12-09T19:06:14,949 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 696259db58514fa9b861f8e84850f546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733771172859 2024-12-09T19:06:14,951 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/A of cfa5a71b9d5357947521a419cca3f2d4 into 1f49d227ac3d457c832295871b97a7a6(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:14,951 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:14,951 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/A, priority=13, startTime=1733771174525; duration=0sec 2024-12-09T19:06:14,951 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:14,952 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:A 2024-12-09T19:06:14,959 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#C#compaction#441 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:14,960 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/8ab240bdacc947528720a86bfca943a7 is 50, key is test_row_0/C:col10/1733771172867/Put/seqid=0 2024-12-09T19:06:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742349_1525 (size=12697) 2024-12-09T19:06:14,980 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/8ab240bdacc947528720a86bfca943a7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/8ab240bdacc947528720a86bfca943a7 2024-12-09T19:06:14,985 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/C of cfa5a71b9d5357947521a419cca3f2d4 into 8ab240bdacc947528720a86bfca943a7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:14,985 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:14,985 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/C, priority=13, startTime=1733771174525; duration=0sec 2024-12-09T19:06:14,985 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:14,985 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:C 2024-12-09T19:06:14,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/01ea95014a7a4eafaac5636b68356929 2024-12-09T19:06:14,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/a25ad4b2416c4938b4c0644ef2a840aa is 50, key is test_row_0/B:col10/1733771172900/Put/seqid=0 2024-12-09T19:06:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742350_1526 (size=12151) 2024-12-09T19:06:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:15,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:15,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771235062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771235064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771235064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771235064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771235064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,067 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771235174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771235174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771235174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771235385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771235385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771235385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,395 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/a25ad4b2416c4938b4c0644ef2a840aa 2024-12-09T19:06:15,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/177c4c4a675b4e4baac0b94ed85804e6 is 50, key is test_row_0/C:col10/1733771172900/Put/seqid=0 2024-12-09T19:06:15,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742351_1527 (size=12151) 2024-12-09T19:06:15,404 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/177c4c4a675b4e4baac0b94ed85804e6 2024-12-09T19:06:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/01ea95014a7a4eafaac5636b68356929 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929 2024-12-09T19:06:15,413 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929, entries=150, sequenceid=249, filesize=30.4 K 2024-12-09T19:06:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/a25ad4b2416c4938b4c0644ef2a840aa as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/a25ad4b2416c4938b4c0644ef2a840aa 2024-12-09T19:06:15,416 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/a25ad4b2416c4938b4c0644ef2a840aa, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T19:06:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/177c4c4a675b4e4baac0b94ed85804e6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/177c4c4a675b4e4baac0b94ed85804e6 2024-12-09T19:06:15,419 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/177c4c4a675b4e4baac0b94ed85804e6, entries=150, sequenceid=249, filesize=11.9 K 2024-12-09T19:06:15,420 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for cfa5a71b9d5357947521a419cca3f2d4 in 851ms, sequenceid=249, compaction requested=false 2024-12-09T19:06:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-09T19:06:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-09T19:06:15,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-09T19:06:15,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5280 sec 2024-12-09T19:06:15,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.5380 sec 2024-12-09T19:06:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:15,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:06:15,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094d40006a34f64bc1a3274686b73bafb0_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:15,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771235717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771235718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771235718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742352_1528 (size=12454) 2024-12-09T19:06:15,727 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:15,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094d40006a34f64bc1a3274686b73bafb0_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094d40006a34f64bc1a3274686b73bafb0_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:15,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/fe91370e44804e2ca8448f0d860893ba, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:15,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/fe91370e44804e2ca8448f0d860893ba is 175, key is test_row_0/A:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:15,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742353_1529 (size=31255) 2024-12-09T19:06:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771235825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771235825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771235825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:15,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771236034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771236034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771236034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,142 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/fe91370e44804e2ca8448f0d860893ba 2024-12-09T19:06:16,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/d1ac7b510ba74646b427b59e7dbdb34c is 50, key is test_row_0/B:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742354_1530 (size=12301) 2024-12-09T19:06:16,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771236345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771236345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771236345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/d1ac7b510ba74646b427b59e7dbdb34c 2024-12-09T19:06:16,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/09257bed225f49999d2fa2c417b53298 is 50, key is test_row_0/C:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:16,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742355_1531 (size=12301) 2024-12-09T19:06:16,700 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T19:06:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771236854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771236854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771236854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:16,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/09257bed225f49999d2fa2c417b53298 2024-12-09T19:06:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/fe91370e44804e2ca8448f0d860893ba as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba 2024-12-09T19:06:16,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba, entries=150, sequenceid=266, filesize=30.5 K 2024-12-09T19:06:16,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/d1ac7b510ba74646b427b59e7dbdb34c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/d1ac7b510ba74646b427b59e7dbdb34c 2024-12-09T19:06:16,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/d1ac7b510ba74646b427b59e7dbdb34c, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T19:06:16,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/09257bed225f49999d2fa2c417b53298 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/09257bed225f49999d2fa2c417b53298 2024-12-09T19:06:16,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/09257bed225f49999d2fa2c417b53298, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T19:06:16,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for cfa5a71b9d5357947521a419cca3f2d4 in 1275ms, sequenceid=266, compaction requested=true 2024-12-09T19:06:16,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T19:06:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:16,972 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T19:06:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:16,972 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfa5a71b9d5357947521a419cca3f2d4:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T19:06:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:16,972 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:16,972 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:16,972 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/A is initiating minor compaction (all files) 2024-12-09T19:06:16,972 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/B is initiating minor compaction (all files) 2024-12-09T19:06:16,972 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/B in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:16,972 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/A in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:16,973 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/1f49d227ac3d457c832295871b97a7a6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=91.8 K 2024-12-09T19:06:16,973 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c145e658e1947cfa22cb15a8040e80b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/a25ad4b2416c4938b4c0644ef2a840aa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/d1ac7b510ba74646b427b59e7dbdb34c] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=36.3 K 2024-12-09T19:06:16,973 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. files: [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/1f49d227ac3d457c832295871b97a7a6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba] 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c145e658e1947cfa22cb15a8040e80b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733771172859 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f49d227ac3d457c832295871b97a7a6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733771172859 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting a25ad4b2416c4938b4c0644ef2a840aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733771172900 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01ea95014a7a4eafaac5636b68356929, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733771172900 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ac7b510ba74646b427b59e7dbdb34c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733771175063 2024-12-09T19:06:16,973 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe91370e44804e2ca8448f0d860893ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733771175063 2024-12-09T19:06:16,978 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#B#compaction#447 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:16,979 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/15b5de36607a4fe090a9e6d9ad147eb3 is 50, key is test_row_0/B:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:16,980 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:16,982 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412098702cc7374c945e287b87c2e435ecb4e_cfa5a71b9d5357947521a419cca3f2d4 store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:16,983 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412098702cc7374c945e287b87c2e435ecb4e_cfa5a71b9d5357947521a419cca3f2d4, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:16,983 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098702cc7374c945e287b87c2e435ecb4e_cfa5a71b9d5357947521a419cca3f2d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:16,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742357_1533 (size=4469) 2024-12-09T19:06:17,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742356_1532 (size=12949) 2024-12-09T19:06:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-09T19:06:17,054 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:17,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T19:06:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T19:06:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:17,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T19:06:17,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T19:06:17,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T19:06:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:17,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-09T19:06:17,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-09T19:06:17,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209193a3523d8e24bdd9b60242a277a66e7_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771175711/Put/seqid=0 2024-12-09T19:06:17,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742358_1534 (size=14994) 2024-12-09T19:06:17,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:17,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209193a3523d8e24bdd9b60242a277a66e7_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209193a3523d8e24bdd9b60242a277a66e7_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:17,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/4374fa46917d4eeb96573fd8a6f11602, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:17,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/4374fa46917d4eeb96573fd8a6f11602 is 175, key is test_row_0/A:col10/1733771175711/Put/seqid=0 2024-12-09T19:06:17,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742359_1535 (size=39949) 2024-12-09T19:06:17,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771237110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771237110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:17,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:17,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771237224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771237224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:17,399 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#A#compaction#448 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:17,399 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/e65f7eefaa7e495c8dd16ea231d86259 is 175, key is test_row_0/A:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:17,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742360_1536 (size=31903) 2024-12-09T19:06:17,412 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/15b5de36607a4fe090a9e6d9ad147eb3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/15b5de36607a4fe090a9e6d9ad147eb3 2024-12-09T19:06:17,415 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/B of cfa5a71b9d5357947521a419cca3f2d4 into 15b5de36607a4fe090a9e6d9ad147eb3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:17,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:17,415 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/B, priority=13, startTime=1733771176972; duration=0sec 2024-12-09T19:06:17,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T19:06:17,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:B 2024-12-09T19:06:17,415 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T19:06:17,416 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T19:06:17,416 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1541): cfa5a71b9d5357947521a419cca3f2d4/C is initiating minor compaction (all files) 2024-12-09T19:06:17,416 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfa5a71b9d5357947521a419cca3f2d4/C in TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,416 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/8ab240bdacc947528720a86bfca943a7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/177c4c4a675b4e4baac0b94ed85804e6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/09257bed225f49999d2fa2c417b53298] into tmpdir=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp, totalSize=36.3 K 2024-12-09T19:06:17,417 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab240bdacc947528720a86bfca943a7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733771172859 2024-12-09T19:06:17,417 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 177c4c4a675b4e4baac0b94ed85804e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733771172900 2024-12-09T19:06:17,417 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] compactions.Compactor(225): Compacting 09257bed225f49999d2fa2c417b53298, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733771175063 2024-12-09T19:06:17,422 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfa5a71b9d5357947521a419cca3f2d4#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T19:06:17,422 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/3b0b4147c44b4d80974ab3c32bb7ee53 is 50, key is test_row_0/C:col10/1733771175695/Put/seqid=0 2024-12-09T19:06:17,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742361_1537 (size=12949) 2024-12-09T19:06:17,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771237435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771237435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:17,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/4374fa46917d4eeb96573fd8a6f11602 2024-12-09T19:06:17,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/c3c7a20729a74749945a8528c33edf9c is 50, key is test_row_0/B:col10/1733771175711/Put/seqid=0 2024-12-09T19:06:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742362_1538 (size=12301) 2024-12-09T19:06:17,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:17,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771237744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771237744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:17,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:17,850 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/e65f7eefaa7e495c8dd16ea231d86259 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e65f7eefaa7e495c8dd16ea231d86259 2024-12-09T19:06:17,852 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/3b0b4147c44b4d80974ab3c32bb7ee53 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/3b0b4147c44b4d80974ab3c32bb7ee53 2024-12-09T19:06:17,854 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/A of cfa5a71b9d5357947521a419cca3f2d4 into e65f7eefaa7e495c8dd16ea231d86259(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:17,854 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:17,854 INFO [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/A, priority=13, startTime=1733771176972; duration=0sec 2024-12-09T19:06:17,854 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:17,854 DEBUG [RS:0;a437f8b9ba7d:46189-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:A 2024-12-09T19:06:17,856 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfa5a71b9d5357947521a419cca3f2d4/C of cfa5a71b9d5357947521a419cca3f2d4 into 3b0b4147c44b4d80974ab3c32bb7ee53(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T19:06:17,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:17,856 INFO [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., storeName=cfa5a71b9d5357947521a419cca3f2d4/C, priority=13, startTime=1733771176972; duration=0sec 2024-12-09T19:06:17,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T19:06:17,856 DEBUG [RS:0;a437f8b9ba7d:46189-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfa5a71b9d5357947521a419cca3f2d4:C 2024-12-09T19:06:17,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53358 deadline: 1733771237864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1733771237864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,865 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_2' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,865 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_0' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53378 deadline: 1733771237874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:17,876 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a437f8b9ba7d,46189,1733770998237 for 'test_row_1' in cfa5a71b9d5357947521a419cca3f2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T19:06:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:17,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/c3c7a20729a74749945a8528c33edf9c 2024-12-09T19:06:17,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/a108a10e95184934b7ea51ac9adf8c4f is 50, key is test_row_0/C:col10/1733771175711/Put/seqid=0 2024-12-09T19:06:17,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742363_1539 (size=12301) 2024-12-09T19:06:17,968 INFO [Thread-2084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2084. 2024-12-09T19:06:17,968 DEBUG [Thread-2084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:17,968 DEBUG [Thread-2084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,969 DEBUG [Thread-2084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:17,970 INFO [Thread-2078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2078. 2024-12-09T19:06:17,970 DEBUG [Thread-2078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:17,970 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,970 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:17,970 INFO [Thread-2086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2086. 2024-12-09T19:06:17,970 DEBUG [Thread-2086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:17,970 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,970 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:17,971 INFO [Thread-2082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2082. 2024-12-09T19:06:17,971 DEBUG [Thread-2082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:17,971 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,972 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:17,972 INFO [Thread-2080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2080. 2024-12-09T19:06:17,972 DEBUG [Thread-2080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:17,972 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,972 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:17,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:17,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:17,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:18,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T19:06:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53414 deadline: 1733771238255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53390 deadline: 1733771238255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 , the old value is region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T19:06:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., hostname=a437f8b9ba7d,46189,1733770998237, seqNum=5 because the exception is null or not the one we care about 2024-12-09T19:06:18,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:18,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T19:06:18,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/a108a10e95184934b7ea51ac9adf8c4f 2024-12-09T19:06:18,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/4374fa46917d4eeb96573fd8a6f11602 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/4374fa46917d4eeb96573fd8a6f11602 2024-12-09T19:06:18,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/4374fa46917d4eeb96573fd8a6f11602, entries=200, sequenceid=289, filesize=39.0 K 2024-12-09T19:06:18,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/c3c7a20729a74749945a8528c33edf9c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/c3c7a20729a74749945a8528c33edf9c 2024-12-09T19:06:18,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/c3c7a20729a74749945a8528c33edf9c, entries=150, sequenceid=289, filesize=12.0 K 2024-12-09T19:06:18,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/a108a10e95184934b7ea51ac9adf8c4f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/a108a10e95184934b7ea51ac9adf8c4f 2024-12-09T19:06:18,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/a108a10e95184934b7ea51ac9adf8c4f, entries=150, sequenceid=289, filesize=12.0 K 2024-12-09T19:06:18,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for cfa5a71b9d5357947521a419cca3f2d4 in 1269ms, sequenceid=289, compaction requested=false 2024-12-09T19:06:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:18,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46189 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T19:06:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:18,469 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-09T19:06:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d6a66cd12efe4589b36a6bbdce84224e_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771177109/Put/seqid=0 2024-12-09T19:06:18,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742364_1540 (size=12454) 2024-12-09T19:06:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:18,895 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d6a66cd12efe4589b36a6bbdce84224e_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d6a66cd12efe4589b36a6bbdce84224e_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:18,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/df46dd016a2348929508f69b774e4f9f, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:18,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/df46dd016a2348929508f69b774e4f9f is 175, key is test_row_0/A:col10/1733771177109/Put/seqid=0 2024-12-09T19:06:18,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742365_1541 (size=31255) 2024-12-09T19:06:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46189 {}] regionserver.HRegion(8855): Flush requested on cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:19,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. as already flushing 2024-12-09T19:06:19,269 INFO [Thread-2073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2073. 2024-12-09T19:06:19,269 DEBUG [Thread-2073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:19,269 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,269 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:19,277 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-12-09T19:06:19,277 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:19,277 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,277 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:19,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/df46dd016a2348929508f69b774e4f9f 2024-12-09T19:06:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/8f35a9f9769e4e1380b9adb921f84504 is 50, key is test_row_0/B:col10/1733771177109/Put/seqid=0 2024-12-09T19:06:19,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742366_1542 (size=12301) 2024-12-09T19:06:19,316 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/8f35a9f9769e4e1380b9adb921f84504 2024-12-09T19:06:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/71d406a66ddf475ab1d15db84818dcd3 is 50, key is test_row_0/C:col10/1733771177109/Put/seqid=0 2024-12-09T19:06:19,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742367_1543 (size=12301) 2024-12-09T19:06:19,728 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/71d406a66ddf475ab1d15db84818dcd3 2024-12-09T19:06:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/df46dd016a2348929508f69b774e4f9f as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/df46dd016a2348929508f69b774e4f9f 2024-12-09T19:06:19,743 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/df46dd016a2348929508f69b774e4f9f, entries=150, sequenceid=305, filesize=30.5 K 2024-12-09T19:06:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/8f35a9f9769e4e1380b9adb921f84504 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/8f35a9f9769e4e1380b9adb921f84504 2024-12-09T19:06:19,747 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/8f35a9f9769e4e1380b9adb921f84504, entries=150, sequenceid=305, filesize=12.0 K 2024-12-09T19:06:19,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/71d406a66ddf475ab1d15db84818dcd3 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/71d406a66ddf475ab1d15db84818dcd3 2024-12-09T19:06:19,752 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/71d406a66ddf475ab1d15db84818dcd3, entries=150, sequenceid=305, filesize=12.0 K 2024-12-09T19:06:19,753 INFO [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for cfa5a71b9d5357947521a419cca3f2d4 in 1284ms, sequenceid=305, compaction requested=true 2024-12-09T19:06:19,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for cfa5a71b9d5357947521a419cca3f2d4: 2024-12-09T19:06:19,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:19,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T19:06:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T19:06:19,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T19:06:19,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6990 sec 2024-12-09T19:06:19,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.7030 sec 2024-12-09T19:06:19,878 INFO [Thread-2075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2075. 2024-12-09T19:06:19,878 DEBUG [Thread-2075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:19,878 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,878 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:19,887 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-12-09T19:06:19,887 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:19,887 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,887 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:19,895 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-12-09T19:06:19,895 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T19:06:19,895 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,896 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:19,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T19:06:21,226 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5134 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5008 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5157 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5127 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5000 2024-12-09T19:06:21,226 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T19:06:21,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:06:21,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:06:21,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:21,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:21,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:21,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T19:06:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T19:06:21,236 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771181236"}]},"ts":"1733771181236"} 2024-12-09T19:06:21,238 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T19:06:21,238 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T19:06:21,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T19:06:21,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, UNASSIGN}] 2024-12-09T19:06:21,241 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, UNASSIGN 2024-12-09T19:06:21,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=CLOSING, regionLocation=a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:21,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, UNASSIGN because future has completed 2024-12-09T19:06:21,243 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T19:06:21,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237}] 2024-12-09T19:06:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T19:06:21,397 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(122): Close cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:21,397 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T19:06:21,397 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1722): Closing cfa5a71b9d5357947521a419cca3f2d4, disabling compactions & flushes 2024-12-09T19:06:21,397 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:21,398 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:21,398 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. after waiting 0 ms 2024-12-09T19:06:21,398 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:21,398 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(2902): Flushing cfa5a71b9d5357947521a419cca3f2d4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-09T19:06:21,408 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093f6a8d2f19b64556a9b839c98377fbdb_cfa5a71b9d5357947521a419cca3f2d4 is 50, key is test_row_0/A:col10/1733771179875/Put/seqid=0 2024-12-09T19:06:21,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742368_1544 (size=12454) 2024-12-09T19:06:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T19:06:21,813 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T19:06:21,823 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093f6a8d2f19b64556a9b839c98377fbdb_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093f6a8d2f19b64556a9b839c98377fbdb_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:21,825 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/cc13feb44c9f4d5eb4c0f602386762e8, store: [table=TestAcidGuarantees family=A region=cfa5a71b9d5357947521a419cca3f2d4] 2024-12-09T19:06:21,827 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/cc13feb44c9f4d5eb4c0f602386762e8 is 175, key is test_row_0/A:col10/1733771179875/Put/seqid=0 2024-12-09T19:06:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742369_1545 (size=31255) 2024-12-09T19:06:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T19:06:22,232 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/cc13feb44c9f4d5eb4c0f602386762e8 2024-12-09T19:06:22,244 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/572b443ab23d4bd4a14c4874ff7f767d is 50, key is test_row_0/B:col10/1733771179875/Put/seqid=0 2024-12-09T19:06:22,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742370_1546 (size=12301) 2024-12-09T19:06:22,248 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/572b443ab23d4bd4a14c4874ff7f767d 2024-12-09T19:06:22,255 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/c17694f263734a839527028b17b51e1c is 50, key is test_row_0/C:col10/1733771179875/Put/seqid=0 2024-12-09T19:06:22,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742371_1547 (size=12301) 2024-12-09T19:06:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T19:06:22,660 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/c17694f263734a839527028b17b51e1c 2024-12-09T19:06:22,671 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/A/cc13feb44c9f4d5eb4c0f602386762e8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/cc13feb44c9f4d5eb4c0f602386762e8 2024-12-09T19:06:22,677 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/cc13feb44c9f4d5eb4c0f602386762e8, entries=150, sequenceid=313, filesize=30.5 K 2024-12-09T19:06:22,677 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/B/572b443ab23d4bd4a14c4874ff7f767d as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/572b443ab23d4bd4a14c4874ff7f767d 2024-12-09T19:06:22,681 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/572b443ab23d4bd4a14c4874ff7f767d, entries=150, sequenceid=313, filesize=12.0 K 2024-12-09T19:06:22,681 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/.tmp/C/c17694f263734a839527028b17b51e1c as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/c17694f263734a839527028b17b51e1c 2024-12-09T19:06:22,683 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/c17694f263734a839527028b17b51e1c, entries=150, sequenceid=313, filesize=12.0 K 2024-12-09T19:06:22,684 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for cfa5a71b9d5357947521a419cca3f2d4 in 1286ms, sequenceid=313, compaction requested=true 2024-12-09T19:06:22,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/10772f0f3de04f3cb8914639eee7a2e1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/6fb8cae62a4f42c89dd3e820d75fcc28, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/136149b126b941898b67045352f7be88, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e208b22f19b14c69bab461c8ced6e2d2, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/1f49d227ac3d457c832295871b97a7a6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba] to archive 2024-12-09T19:06:22,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:06:22,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/a93d9abd609e4c8a8294fe75a0667be2 2024-12-09T19:06:22,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c509290b24444913a6d12d9be9aa84c5 2024-12-09T19:06:22,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/51b6ccf8a43d45ae8b1c200638743024 2024-12-09T19:06:22,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/10772f0f3de04f3cb8914639eee7a2e1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/10772f0f3de04f3cb8914639eee7a2e1 2024-12-09T19:06:22,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/3e15a21c1f294227b74faefeb73555ea 2024-12-09T19:06:22,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/59742770696a4cc3bc06ebe0d15beb13 2024-12-09T19:06:22,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/6fb8cae62a4f42c89dd3e820d75fcc28 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/6fb8cae62a4f42c89dd3e820d75fcc28 2024-12-09T19:06:22,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/c7b76b5dadd04f5ea12439f5e3957c20 2024-12-09T19:06:22,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/63c12d85e03e4e179d9cab81bc6d78f8 2024-12-09T19:06:22,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/136149b126b941898b67045352f7be88 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/136149b126b941898b67045352f7be88 2024-12-09T19:06:22,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/d9100d47edb24a3c8864dce30516750b 2024-12-09T19:06:22,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/f43ac91c68ba40779bb065067bbd25b4 2024-12-09T19:06:22,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/077ef55a6bac40c1a2b1c101bd04aea6 2024-12-09T19:06:22,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e208b22f19b14c69bab461c8ced6e2d2 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e208b22f19b14c69bab461c8ced6e2d2 2024-12-09T19:06:22,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/66223ec6c6b54cdfb2c2ad69663d757d 2024-12-09T19:06:22,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/b0fbc91c6ee84d778e8fb367d6e39e57 2024-12-09T19:06:22,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/1f49d227ac3d457c832295871b97a7a6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/1f49d227ac3d457c832295871b97a7a6 2024-12-09T19:06:22,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/01ea95014a7a4eafaac5636b68356929 2024-12-09T19:06:22,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/fe91370e44804e2ca8448f0d860893ba 2024-12-09T19:06:22,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/925daa12a73a4c458d5aa8add821030b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5a2dcf7f836045bf8b6ef42327d8bfae, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/67e5dc4dfd9a436e8c25efd1375f595c, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/ec64aa41a2914f44ba860abde87f3299, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/dbc4690200864185b74feaf80f773771, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/2251e6b003f840a3a865ffe651021c35, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5b265ab34ae34cada6983e4cc5499ccb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/bad3e5cdd816452395b98d40137683c8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/700fdbb2770a48939e149f99d01884ed, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/3a0d3fe34d224a859a5ee18a8a7852c8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/138202461b614fdd84b295e7e401b055, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/9335d39659f740c583c4a26aec14abb4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/b48aa48b5cef4edaa18e86a54c2e77a1, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/94a058d810d1427e8b32cfcf35e14808, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/7b2a219549584a5e82ff0546960e1edb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c145e658e1947cfa22cb15a8040e80b, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c5f4c16e6ab44ccb79f12defcee06e4, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/a25ad4b2416c4938b4c0644ef2a840aa, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/d1ac7b510ba74646b427b59e7dbdb34c] to archive 2024-12-09T19:06:22,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:06:22,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/925daa12a73a4c458d5aa8add821030b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/925daa12a73a4c458d5aa8add821030b 2024-12-09T19:06:22,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5a2dcf7f836045bf8b6ef42327d8bfae to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5a2dcf7f836045bf8b6ef42327d8bfae 2024-12-09T19:06:22,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/67e5dc4dfd9a436e8c25efd1375f595c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/67e5dc4dfd9a436e8c25efd1375f595c 2024-12-09T19:06:22,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/ec64aa41a2914f44ba860abde87f3299 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/ec64aa41a2914f44ba860abde87f3299 2024-12-09T19:06:22,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/dbc4690200864185b74feaf80f773771 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/dbc4690200864185b74feaf80f773771 2024-12-09T19:06:22,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/2251e6b003f840a3a865ffe651021c35 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/2251e6b003f840a3a865ffe651021c35 2024-12-09T19:06:22,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5b265ab34ae34cada6983e4cc5499ccb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/5b265ab34ae34cada6983e4cc5499ccb 2024-12-09T19:06:22,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/bad3e5cdd816452395b98d40137683c8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/bad3e5cdd816452395b98d40137683c8 2024-12-09T19:06:22,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/700fdbb2770a48939e149f99d01884ed to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/700fdbb2770a48939e149f99d01884ed 2024-12-09T19:06:22,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/3a0d3fe34d224a859a5ee18a8a7852c8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/3a0d3fe34d224a859a5ee18a8a7852c8 2024-12-09T19:06:22,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/138202461b614fdd84b295e7e401b055 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/138202461b614fdd84b295e7e401b055 2024-12-09T19:06:22,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/9335d39659f740c583c4a26aec14abb4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/9335d39659f740c583c4a26aec14abb4 2024-12-09T19:06:22,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/b48aa48b5cef4edaa18e86a54c2e77a1 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/b48aa48b5cef4edaa18e86a54c2e77a1 2024-12-09T19:06:22,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/94a058d810d1427e8b32cfcf35e14808 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/94a058d810d1427e8b32cfcf35e14808 2024-12-09T19:06:22,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/7b2a219549584a5e82ff0546960e1edb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/7b2a219549584a5e82ff0546960e1edb 2024-12-09T19:06:22,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c145e658e1947cfa22cb15a8040e80b to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c145e658e1947cfa22cb15a8040e80b 2024-12-09T19:06:22,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c5f4c16e6ab44ccb79f12defcee06e4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/0c5f4c16e6ab44ccb79f12defcee06e4 2024-12-09T19:06:22,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/a25ad4b2416c4938b4c0644ef2a840aa to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/a25ad4b2416c4938b4c0644ef2a840aa 2024-12-09T19:06:22,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/d1ac7b510ba74646b427b59e7dbdb34c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/d1ac7b510ba74646b427b59e7dbdb34c 2024-12-09T19:06:22,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/7d2715d90d704127975497c8af3d649d, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/f18f823aa1e448e5b6420bbf28a852d7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/d473f81e358d4f81a7466767ada34678, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/fb9970668c8e4bd896d7f0f409e4f7fb, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/1bca99463da8424daad26c0ba94b2009, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/538adfcc9a3844088b4b69959be27a00, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/6279f1aca484464d9980496f4fa28581, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/63090fa972f4407aa1a48e5808eba3ae, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/ee6ed7a3ef5a4dc8a705a0ec51a80394, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b7740a295bdd4af5b98fa44482dbc993, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/0d7be3ede7354fc1baf56bbc60bd10fe, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/50ebc493d1a64395adf713ed0534f8c7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/2fd78c19999b4ab591103e3f721484a8, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b2f000e300f24b0bb56897315ced6013, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/36207faf2db5440ab766b5ee5d96b3cc, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/8ab240bdacc947528720a86bfca943a7, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/696259db58514fa9b861f8e84850f546, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/177c4c4a675b4e4baac0b94ed85804e6, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/09257bed225f49999d2fa2c417b53298] to archive 2024-12-09T19:06:22,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T19:06:22,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/7d2715d90d704127975497c8af3d649d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/7d2715d90d704127975497c8af3d649d 2024-12-09T19:06:22,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/f18f823aa1e448e5b6420bbf28a852d7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/f18f823aa1e448e5b6420bbf28a852d7 2024-12-09T19:06:22,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/d473f81e358d4f81a7466767ada34678 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/d473f81e358d4f81a7466767ada34678 2024-12-09T19:06:22,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/fb9970668c8e4bd896d7f0f409e4f7fb to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/fb9970668c8e4bd896d7f0f409e4f7fb 2024-12-09T19:06:22,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/1bca99463da8424daad26c0ba94b2009 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/1bca99463da8424daad26c0ba94b2009 2024-12-09T19:06:22,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/538adfcc9a3844088b4b69959be27a00 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/538adfcc9a3844088b4b69959be27a00 2024-12-09T19:06:22,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/6279f1aca484464d9980496f4fa28581 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/6279f1aca484464d9980496f4fa28581 2024-12-09T19:06:22,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/63090fa972f4407aa1a48e5808eba3ae to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/63090fa972f4407aa1a48e5808eba3ae 2024-12-09T19:06:22,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/ee6ed7a3ef5a4dc8a705a0ec51a80394 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/ee6ed7a3ef5a4dc8a705a0ec51a80394 2024-12-09T19:06:22,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b7740a295bdd4af5b98fa44482dbc993 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b7740a295bdd4af5b98fa44482dbc993 2024-12-09T19:06:22,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/0d7be3ede7354fc1baf56bbc60bd10fe to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/0d7be3ede7354fc1baf56bbc60bd10fe 2024-12-09T19:06:22,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/50ebc493d1a64395adf713ed0534f8c7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/50ebc493d1a64395adf713ed0534f8c7 2024-12-09T19:06:22,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/2fd78c19999b4ab591103e3f721484a8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/2fd78c19999b4ab591103e3f721484a8 2024-12-09T19:06:22,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b2f000e300f24b0bb56897315ced6013 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/b2f000e300f24b0bb56897315ced6013 2024-12-09T19:06:22,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/36207faf2db5440ab766b5ee5d96b3cc to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/36207faf2db5440ab766b5ee5d96b3cc 2024-12-09T19:06:22,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/8ab240bdacc947528720a86bfca943a7 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/8ab240bdacc947528720a86bfca943a7 2024-12-09T19:06:22,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/696259db58514fa9b861f8e84850f546 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/696259db58514fa9b861f8e84850f546 2024-12-09T19:06:22,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/177c4c4a675b4e4baac0b94ed85804e6 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/177c4c4a675b4e4baac0b94ed85804e6 2024-12-09T19:06:22,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/09257bed225f49999d2fa2c417b53298 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/09257bed225f49999d2fa2c417b53298 2024-12-09T19:06:22,733 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/recovered.edits/316.seqid, newMaxSeqId=316, maxSeqId=4 2024-12-09T19:06:22,733 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. 2024-12-09T19:06:22,733 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1676): Region close journal for cfa5a71b9d5357947521a419cca3f2d4: Waiting for close lock at 1733771181397Running coprocessor pre-close hooks at 1733771181397Disabling compacts and flushes for region at 1733771181397Disabling writes for close at 1733771181398 (+1 ms)Obtaining lock to block concurrent updates at 1733771181398Preparing flush snapshotting stores in cfa5a71b9d5357947521a419cca3f2d4 at 1733771181398Finished memstore snapshotting TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733771181399 (+1 ms)Flushing stores of TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4. at 1733771181400 (+1 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/A: creating writer at 1733771181400Flushing mob file cfa5a71b9d5357947521a419cca3f2d4/A: appending metadata at 1733771181408 (+8 ms)Flushing mob file cfa5a71b9d5357947521a419cca3f2d4/A: closing flushed file at 1733771181408Flushing cfa5a71b9d5357947521a419cca3f2d4/A: appending metadata at 1733771181825 (+417 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/A: closing flushed file at 1733771181826 (+1 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/B: creating writer at 1733771182241 (+415 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/B: appending metadata at 1733771182244 (+3 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/B: closing flushed file at 1733771182244Flushing cfa5a71b9d5357947521a419cca3f2d4/C: creating writer at 1733771182252 (+8 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/C: appending metadata at 1733771182255 (+3 ms)Flushing cfa5a71b9d5357947521a419cca3f2d4/C: closing flushed file at 1733771182255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6702071: reopening flushed file at 1733771182670 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a3f4ab1: reopening flushed file at 1733771182677 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f66da8e: reopening flushed file at 1733771182681 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for cfa5a71b9d5357947521a419cca3f2d4 in 1286ms, sequenceid=313, compaction requested=true at 1733771182684 (+3 ms)Writing region close event to WAL at 1733771182731 (+47 ms)Running coprocessor post-close hooks at 1733771182733 (+2 ms)Closed at 1733771182733 2024-12-09T19:06:22,735 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(157): Closed cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:22,735 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=cfa5a71b9d5357947521a419cca3f2d4, regionState=CLOSED 2024-12-09T19:06:22,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 because future has completed 2024-12-09T19:06:22,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T19:06:22,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseRegionProcedure cfa5a71b9d5357947521a419cca3f2d4, server=a437f8b9ba7d,46189,1733770998237 in 1.4930 sec 2024-12-09T19:06:22,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-12-09T19:06:22,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfa5a71b9d5357947521a419cca3f2d4, UNASSIGN in 1.4980 sec 2024-12-09T19:06:22,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T19:06:22,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5000 sec 2024-12-09T19:06:22,741 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733771182741"}]},"ts":"1733771182741"} 2024-12-09T19:06:22,742 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T19:06:22,742 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T19:06:22,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5130 sec 2024-12-09T19:06:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T19:06:23,385 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:23,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T19:06:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:23,388 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T19:06:23,389 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=158, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:23,392 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C, FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/recovered.edits] 2024-12-09T19:06:23,401 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/4374fa46917d4eeb96573fd8a6f11602 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/4374fa46917d4eeb96573fd8a6f11602 2024-12-09T19:06:23,403 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/cc13feb44c9f4d5eb4c0f602386762e8 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/cc13feb44c9f4d5eb4c0f602386762e8 2024-12-09T19:06:23,406 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/df46dd016a2348929508f69b774e4f9f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/df46dd016a2348929508f69b774e4f9f 2024-12-09T19:06:23,408 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e65f7eefaa7e495c8dd16ea231d86259 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/A/e65f7eefaa7e495c8dd16ea231d86259 2024-12-09T19:06:23,411 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/15b5de36607a4fe090a9e6d9ad147eb3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/15b5de36607a4fe090a9e6d9ad147eb3 2024-12-09T19:06:23,413 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/572b443ab23d4bd4a14c4874ff7f767d to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/572b443ab23d4bd4a14c4874ff7f767d 2024-12-09T19:06:23,415 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/8f35a9f9769e4e1380b9adb921f84504 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/8f35a9f9769e4e1380b9adb921f84504 2024-12-09T19:06:23,417 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/c3c7a20729a74749945a8528c33edf9c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/B/c3c7a20729a74749945a8528c33edf9c 2024-12-09T19:06:23,419 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/3b0b4147c44b4d80974ab3c32bb7ee53 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/3b0b4147c44b4d80974ab3c32bb7ee53 2024-12-09T19:06:23,420 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/71d406a66ddf475ab1d15db84818dcd3 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/71d406a66ddf475ab1d15db84818dcd3 2024-12-09T19:06:23,421 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/a108a10e95184934b7ea51ac9adf8c4f to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/a108a10e95184934b7ea51ac9adf8c4f 2024-12-09T19:06:23,422 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/c17694f263734a839527028b17b51e1c to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/C/c17694f263734a839527028b17b51e1c 2024-12-09T19:06:23,423 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/recovered.edits/316.seqid to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4/recovered.edits/316.seqid 2024-12-09T19:06:23,424 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/default/TestAcidGuarantees/cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,424 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T19:06:23,424 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T19:06:23,425 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T19:06:23,426 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209193a3523d8e24bdd9b60242a277a66e7_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209193a3523d8e24bdd9b60242a277a66e7_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,427 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120927db9b26e0c142efba645bda42fd6f2f_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120927db9b26e0c142efba645bda42fd6f2f_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,428 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093f6a8d2f19b64556a9b839c98377fbdb_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412093f6a8d2f19b64556a9b839c98377fbdb_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,429 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209445954af124e4007a2253c22d15cd50c_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209445954af124e4007a2253c22d15cd50c_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,429 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094d40006a34f64bc1a3274686b73bafb0_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094d40006a34f64bc1a3274686b73bafb0_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,430 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094eb9a987e72b4a5e8097a5c9dcdaaddc_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094eb9a987e72b4a5e8097a5c9dcdaaddc_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,431 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d9fe7386837454f86d4dfb975aa560a_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095d9fe7386837454f86d4dfb975aa560a_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,432 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979d6348652fb4a97955ffc98106214ce_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120979d6348652fb4a97955ffc98106214ce_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,433 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120984446224e47f4e9d9f18e9a8727657d3_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120984446224e47f4e9d9f18e9a8727657d3_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,434 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120989424fd23ed84e4cbc06f42b75b05111_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120989424fd23ed84e4cbc06f42b75b05111_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,435 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d1aa228e6fd49228a8454d9d95bfb08_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099d1aa228e6fd49228a8454d9d95bfb08_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,436 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a0527c6f41ed48a88c5d2d8f8ebd6125_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209a0527c6f41ed48a88c5d2d8f8ebd6125_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,437 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c72776b2d24e4b6a8b13790adf5f8f9f_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c72776b2d24e4b6a8b13790adf5f8f9f_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,438 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d6a66cd12efe4589b36a6bbdce84224e_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d6a66cd12efe4589b36a6bbdce84224e_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,440 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d91f7f2e21cd4fe78848a2fddf4d8b86_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d91f7f2e21cd4fe78848a2fddf4d8b86_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,441 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e35b33ec3439446090c355720db69a5f_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e35b33ec3439446090c355720db69a5f_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,442 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e3f934c6c8714f87b2ce4a34ca1894e3_cfa5a71b9d5357947521a419cca3f2d4 to hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209e3f934c6c8714f87b2ce4a34ca1894e3_cfa5a71b9d5357947521a419cca3f2d4 2024-12-09T19:06:23,442 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T19:06:23,445 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=158, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:23,448 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T19:06:23,450 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T19:06:23,451 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=158, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:23,451 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T19:06:23,451 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733771183451"}]},"ts":"9223372036854775807"} 2024-12-09T19:06:23,453 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T19:06:23,453 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cfa5a71b9d5357947521a419cca3f2d4, NAME => 'TestAcidGuarantees,,1733771154751.cfa5a71b9d5357947521a419cca3f2d4.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T19:06:23,453 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T19:06:23,453 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733771183453"}]},"ts":"9223372036854775807"} 2024-12-09T19:06:23,454 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T19:06:23,455 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=158, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T19:06:23,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-12-09T19:06:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T19:06:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T19:06:23,495 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T19:06:23,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 240) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=349 (was 324) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2125 (was 2148) 2024-12-09T19:06:23,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T19:06:23,511 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:06:23,511 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:06:23,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:23,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:23,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T19:06:23,511 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T19:06:23,511 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1432429954, stopped=false 2024-12-09T19:06:23,511 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a437f8b9ba7d,38053,1733770997561 2024-12-09T19:06:23,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T19:06:23,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T19:06:23,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:06:23,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:06:23,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T19:06:23,558 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T19:06:23,559 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:06:23,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:23,559 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T19:06:23,559 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T19:06:23,560 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a437f8b9ba7d,46189,1733770998237' ***** 2024-12-09T19:06:23,561 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T19:06:23,561 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T19:06:23,562 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T19:06:23,562 INFO [RS:0;a437f8b9ba7d:46189 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T19:06:23,562 INFO [RS:0;a437f8b9ba7d:46189 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T19:06:23,562 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(959): stopping server a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:23,562 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T19:06:23,562 INFO [RS:0;a437f8b9ba7d:46189 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a437f8b9ba7d:46189. 2024-12-09T19:06:23,562 DEBUG [RS:0;a437f8b9ba7d:46189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T19:06:23,562 DEBUG [RS:0;a437f8b9ba7d:46189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:23,563 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T19:06:23,563 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T19:06:23,563 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T19:06:23,563 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T19:06:23,564 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T19:06:23,564 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T19:06:23,564 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:23,564 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T19:06:23,564 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T19:06:23,564 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T19:06:23,564 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T19:06:23,564 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T19:06:23,564 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T19:06:23,584 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/info/befb904502e649c2a564edf3016d514a is 95, key is TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727./info:/1733771033369/DeleteFamily/seqid=0 2024-12-09T19:06:23,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742372_1548 (size=6514) 2024-12-09T19:06:23,645 INFO [regionserver/a437f8b9ba7d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T19:06:23,764 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:23,965 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:23,988 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/info/befb904502e649c2a564edf3016d514a 2024-12-09T19:06:24,008 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/ns/03265f46b8aa4723b182e150d196f5f6 is 93, key is TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727./ns:/1733771033335/DeleteFamily/seqid=0 2024-12-09T19:06:24,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742373_1549 (size=5906) 2024-12-09T19:06:24,074 INFO [regionserver/a437f8b9ba7d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T19:06:24,074 INFO [regionserver/a437f8b9ba7d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T19:06:24,165 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:24,365 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:24,411 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/ns/03265f46b8aa4723b182e150d196f5f6 2024-12-09T19:06:24,429 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/rep_barrier/5e61ff6cc5204454a121052f3c37e7ae is 102, key is TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727./rep_barrier:/1733771033335/DeleteFamily/seqid=0 2024-12-09T19:06:24,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742374_1550 (size=6025) 2024-12-09T19:06:24,565 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:24,766 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:24,834 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/rep_barrier/5e61ff6cc5204454a121052f3c37e7ae 2024-12-09T19:06:24,862 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/table/0def311eee4e4309a675707aa504c8c6 is 96, key is TestAcidGuarantees,,1733771002635.07169db55e935a0918c76ee552d19727./table:/1733771033335/DeleteFamily/seqid=0 2024-12-09T19:06:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742375_1551 (size=5936) 2024-12-09T19:06:24,966 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:25,166 DEBUG [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T19:06:25,267 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/table/0def311eee4e4309a675707aa504c8c6 2024-12-09T19:06:25,270 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/info/befb904502e649c2a564edf3016d514a as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/info/befb904502e649c2a564edf3016d514a 2024-12-09T19:06:25,272 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/info/befb904502e649c2a564edf3016d514a, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T19:06:25,273 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/ns/03265f46b8aa4723b182e150d196f5f6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/ns/03265f46b8aa4723b182e150d196f5f6 2024-12-09T19:06:25,276 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/ns/03265f46b8aa4723b182e150d196f5f6, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T19:06:25,276 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/rep_barrier/5e61ff6cc5204454a121052f3c37e7ae as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/rep_barrier/5e61ff6cc5204454a121052f3c37e7ae 2024-12-09T19:06:25,278 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/rep_barrier/5e61ff6cc5204454a121052f3c37e7ae, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T19:06:25,279 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/.tmp/table/0def311eee4e4309a675707aa504c8c6 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/table/0def311eee4e4309a675707aa504c8c6 2024-12-09T19:06:25,281 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/table/0def311eee4e4309a675707aa504c8c6, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T19:06:25,282 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1718ms, sequenceid=90, compaction requested=false 2024-12-09T19:06:25,285 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T19:06:25,286 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T19:06:25,286 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T19:06:25,286 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733771183564Running coprocessor pre-close hooks at 1733771183564Disabling compacts and flushes for region at 1733771183564Disabling writes for close at 1733771183564Obtaining lock to block concurrent updates at 1733771183564Preparing flush snapshotting stores in 1588230740 at 1733771183564Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733771183565 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733771183565Flushing 1588230740/info: creating writer at 1733771183566 (+1 ms)Flushing 1588230740/info: appending metadata at 1733771183584 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733771183584Flushing 1588230740/ns: creating writer at 1733771183992 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733771184007 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733771184007Flushing 1588230740/rep_barrier: creating writer at 1733771184414 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733771184429 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733771184429Flushing 1588230740/table: creating writer at 1733771184844 (+415 ms)Flushing 1588230740/table: appending metadata at 1733771184862 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733771184862Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ca51367: reopening flushed file at 1733771185269 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acce6fa: reopening flushed file at 1733771185272 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2898d0d1: reopening flushed file at 1733771185276 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b78fb2: reopening flushed file at 1733771185278 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1718ms, sequenceid=90, compaction requested=false at 1733771185282 (+4 ms)Writing region close event to WAL at 1733771185283 (+1 ms)Running coprocessor post-close hooks at 1733771185286 (+3 ms)Closed at 1733771185286 2024-12-09T19:06:25,286 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T19:06:25,367 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(976): stopping server a437f8b9ba7d,46189,1733770998237; all regions closed. 2024-12-09T19:06:25,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741834_1010 (size=25344) 2024-12-09T19:06:25,374 DEBUG [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/oldWALs 2024-12-09T19:06:25,375 INFO [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a437f8b9ba7d%2C46189%2C1733770998237.meta:.meta(num 1733771002216) 2024-12-09T19:06:25,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741832_1008 (size=12403807) 2024-12-09T19:06:25,380 DEBUG [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/oldWALs 2024-12-09T19:06:25,380 INFO [RS:0;a437f8b9ba7d:46189 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a437f8b9ba7d%2C46189%2C1733770998237:(num 1733771001224) 2024-12-09T19:06:25,380 DEBUG [RS:0;a437f8b9ba7d:46189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T19:06:25,380 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T19:06:25,380 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T19:06:25,380 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.ChoreService(370): Chore service for: regionserver/a437f8b9ba7d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T19:06:25,381 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T19:06:25,381 INFO [regionserver/a437f8b9ba7d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T19:06:25,382 INFO [RS:0;a437f8b9ba7d:46189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46189 2024-12-09T19:06:25,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a437f8b9ba7d,46189,1733770998237 2024-12-09T19:06:25,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T19:06:25,424 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T19:06:25,426 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a437f8b9ba7d,46189,1733770998237] 2024-12-09T19:06:25,440 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a437f8b9ba7d,46189,1733770998237 already deleted, retry=false 2024-12-09T19:06:25,440 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a437f8b9ba7d,46189,1733770998237 expired; onlineServers=0 2024-12-09T19:06:25,440 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a437f8b9ba7d,38053,1733770997561' ***** 2024-12-09T19:06:25,441 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T19:06:25,441 INFO [M:0;a437f8b9ba7d:38053 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T19:06:25,441 INFO [M:0;a437f8b9ba7d:38053 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T19:06:25,441 DEBUG [M:0;a437f8b9ba7d:38053 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T19:06:25,441 DEBUG [M:0;a437f8b9ba7d:38053 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T19:06:25,441 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T19:06:25,441 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.large.0-1733771000932 {}] cleaner.HFileCleaner(306): Exit Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.large.0-1733771000932,5,FailOnTimeoutGroup] 2024-12-09T19:06:25,441 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.small.0-1733771000932 {}] cleaner.HFileCleaner(306): Exit Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.small.0-1733771000932,5,FailOnTimeoutGroup] 2024-12-09T19:06:25,442 INFO [M:0;a437f8b9ba7d:38053 {}] hbase.ChoreService(370): Chore service for: master/a437f8b9ba7d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T19:06:25,442 INFO [M:0;a437f8b9ba7d:38053 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T19:06:25,442 DEBUG [M:0;a437f8b9ba7d:38053 {}] master.HMaster(1795): Stopping service threads 2024-12-09T19:06:25,443 INFO [M:0;a437f8b9ba7d:38053 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T19:06:25,443 INFO [M:0;a437f8b9ba7d:38053 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T19:06:25,443 ERROR [M:0;a437f8b9ba7d:38053 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-09T19:06:25,444 INFO [M:0;a437f8b9ba7d:38053 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T19:06:25,444 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T19:06:25,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T19:06:25,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T19:06:25,448 DEBUG [M:0;a437f8b9ba7d:38053 {}] zookeeper.ZKUtil(347): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T19:06:25,448 WARN [M:0;a437f8b9ba7d:38053 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T19:06:25,449 INFO [M:0;a437f8b9ba7d:38053 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/.lastflushedseqids 2024-12-09T19:06:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742376_1552 (size=72) 2024-12-09T19:06:25,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T19:06:25,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46189-0x1000c2d089b0001, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T19:06:25,533 INFO [RS:0;a437f8b9ba7d:46189 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T19:06:25,533 INFO [RS:0;a437f8b9ba7d:46189 {}] regionserver.HRegionServer(1031): Exiting; stopping=a437f8b9ba7d,46189,1733770998237; zookeeper connection closed. 2024-12-09T19:06:25,534 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21409630 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21409630 2024-12-09T19:06:25,535 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T19:06:25,860 INFO [M:0;a437f8b9ba7d:38053 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T19:06:25,861 INFO [M:0;a437f8b9ba7d:38053 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T19:06:25,861 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T19:06:25,861 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:06:25,861 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:06:25,862 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T19:06:25,862 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:06:25,862 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=741.36 KB heapSize=910.62 KB 2024-12-09T19:06:25,880 DEBUG [M:0;a437f8b9ba7d:38053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe6720af00234e02b1eda48f1750f3d8 is 82, key is hbase:meta,,1/info:regioninfo/1733771002292/Put/seqid=0 2024-12-09T19:06:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742377_1553 (size=5672) 2024-12-09T19:06:26,284 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2094 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe6720af00234e02b1eda48f1750f3d8 2024-12-09T19:06:26,310 DEBUG [M:0;a437f8b9ba7d:38053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb29289674274c88be9f1bb820ab91f7 is 2276, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1733771036594/Put/seqid=0 2024-12-09T19:06:26,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742378_1554 (size=42099) 2024-12-09T19:06:26,715 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=740.80 KB at sequenceid=2094 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb29289674274c88be9f1bb820ab91f7 2024-12-09T19:06:26,725 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb29289674274c88be9f1bb820ab91f7 2024-12-09T19:06:26,741 DEBUG [M:0;a437f8b9ba7d:38053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8768de8469da4994ae96a3fd9dde5479 is 69, key is a437f8b9ba7d,46189,1733770998237/rs:state/1733771001002/Put/seqid=0 2024-12-09T19:06:26,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073742379_1555 (size=5156) 2024-12-09T19:06:27,145 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2094 (bloomFilter=true), to=hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8768de8469da4994ae96a3fd9dde5479 2024-12-09T19:06:27,154 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe6720af00234e02b1eda48f1750f3d8 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe6720af00234e02b1eda48f1750f3d8 2024-12-09T19:06:27,159 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe6720af00234e02b1eda48f1750f3d8, entries=8, sequenceid=2094, filesize=5.5 K 2024-12-09T19:06:27,160 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb29289674274c88be9f1bb820ab91f7 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb29289674274c88be9f1bb820ab91f7 2024-12-09T19:06:27,164 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb29289674274c88be9f1bb820ab91f7 2024-12-09T19:06:27,165 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb29289674274c88be9f1bb820ab91f7, entries=158, sequenceid=2094, filesize=41.1 K 2024-12-09T19:06:27,165 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8768de8469da4994ae96a3fd9dde5479 as hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8768de8469da4994ae96a3fd9dde5479 2024-12-09T19:06:27,169 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34093/user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8768de8469da4994ae96a3fd9dde5479, entries=1, sequenceid=2094, filesize=5.0 K 2024-12-09T19:06:27,170 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(3140): Finished flush of dataSize ~741.36 KB/759149, heapSize ~910.32 KB/932168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2094, compaction requested=false 2024-12-09T19:06:27,171 INFO [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T19:06:27,172 DEBUG [M:0;a437f8b9ba7d:38053 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733771185861Disabling compacts and flushes for region at 1733771185861Disabling writes for close at 1733771185862 (+1 ms)Obtaining lock to block concurrent updates at 1733771185862Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733771185862Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=759149, getHeapSize=932408, getOffHeapSize=0, getCellsCount=2277 at 1733771185863 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733771185864 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733771185864Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733771185880 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733771185880Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733771186291 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733771186310 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733771186310Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733771186725 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733771186741 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733771186741Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3762f04e: reopening flushed file at 1733771187153 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ce914: reopening flushed file at 1733771187159 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58638507: reopening flushed file at 1733771187165 (+6 ms)Finished flush of dataSize ~741.36 KB/759149, heapSize ~910.32 KB/932168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2094, compaction requested=false at 1733771187170 (+5 ms)Writing region close event to WAL at 1733771187171 (+1 ms)Closed at 1733771187171 2024-12-09T19:06:27,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37109 is added to blk_1073741830_1006 (size=896410) 2024-12-09T19:06:27,176 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5e27c3dd-2d40-ac0c-49ba-ce1cb47b3b0b/MasterData/WALs/a437f8b9ba7d,38053,1733770997561/a437f8b9ba7d%2C38053%2C1733770997561.1733771000325 not finished, retry = 0 2024-12-09T19:06:27,278 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T19:06:27,278 INFO [M:0;a437f8b9ba7d:38053 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T19:06:27,278 INFO [M:0;a437f8b9ba7d:38053 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38053 2024-12-09T19:06:27,278 INFO [M:0;a437f8b9ba7d:38053 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T19:06:27,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T19:06:27,415 INFO [M:0;a437f8b9ba7d:38053 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T19:06:27,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38053-0x1000c2d089b0000, quorum=127.0.0.1:60613, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T19:06:27,461 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T19:06:27,466 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T19:06:27,466 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T19:06:27,466 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T19:06:27,466 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/hadoop.log.dir/,STOPPED} 2024-12-09T19:06:27,469 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T19:06:27,469 WARN [BP-1730054910-172.17.0.2-1733770994308 heartbeating to localhost/127.0.0.1:34093 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T19:06:27,469 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T19:06:27,469 WARN [BP-1730054910-172.17.0.2-1733770994308 heartbeating to localhost/127.0.0.1:34093 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1730054910-172.17.0.2-1733770994308 (Datanode Uuid ac1a6feb-a787-4ca5-acc6-45637ccb9f1c) service to localhost/127.0.0.1:34093 2024-12-09T19:06:27,471 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/data/data1/current/BP-1730054910-172.17.0.2-1733770994308 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T19:06:27,472 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/cluster_f2856010-96f8-87f9-bcdf-a696664ad150/data/data2/current/BP-1730054910-172.17.0.2-1733770994308 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T19:06:27,472 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T19:06:27,478 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T19:06:27,478 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T19:06:27,478 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T19:06:27,478 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T19:06:27,479 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdc58b3c-6028-572b-a10d-fe645c7045a8/hadoop.log.dir/,STOPPED} 2024-12-09T19:06:27,491 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T19:06:27,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down